mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/exif_write.py

@@ -1,11 +1,11 @@
 # pyre-ignore-all-errors[5, 21, 24]
+from __future__ import annotations
 
 import datetime
 import io
 import json
 import logging
 import math
-import typing as T
 from pathlib import Path
 
 import piexif
@@ -15,9 +15,9 @@ LOG = logging.getLogger(__name__)
 
 
 class ExifEdit:
-    _filename_or_bytes: T.Union[str, bytes]
+    _filename_or_bytes: str | bytes
 
-    def __init__(self, filename_or_bytes: T.Union[Path, bytes]) -> None:
+    def __init__(self, filename_or_bytes: Path | bytes) -> None:
         """Initialize the object"""
         if isinstance(filename_or_bytes, Path):
             # make sure filename is resolved to avoid to be interpretted as bytes in piexif
@@ -25,12 +25,12 @@ class ExifEdit:
             self._filename_or_bytes = str(filename_or_bytes.resolve())
         else:
             self._filename_or_bytes = filename_or_bytes
-        self._ef: T.Dict = piexif.load(self._filename_or_bytes)
+        self._ef: dict = piexif.load(self._filename_or_bytes)
 
     @staticmethod
     def decimal_to_dms(
         value: float, precision: int
-    ) -> T.Tuple[T.Tuple[float, int], T.Tuple[float, int], T.Tuple[float, int]]:
+    ) -> tuple[tuple[float, int], tuple[float, int], tuple[float, int]]:
         """
         Convert decimal position to degrees, minutes, seconds in a fromat supported by EXIF
         """
@@ -40,9 +40,11 @@ class ExifEdit:
 
         return (deg, 1), (min, 1), (sec, precision)
 
-    def add_image_description(self, data: T.Dict) -> None:
+    def add_image_description(self, data: dict) -> None:
         """Add a dict to image description."""
-        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
+        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(
+            data, sort_keys=True, separators=(",", ":")
+        )
 
     def add_orientation(self, orientation: int) -> None:
         """Add image orientation to image."""
@@ -201,7 +203,7 @@ class ExifEdit:
             piexif.insert(exif_bytes, self._filename_or_bytes, output)
             return output.read()
 
-    def write(self, filename: T.Optional[Path] = None) -> None:
+    def write(self, filename: Path | None = None) -> None:
         """Save exif data to file."""
         if filename is None:
             if not isinstance(self._filename_or_bytes, str):

mapillary_tools/exiftool_read.py

@@ -1,13 +1,15 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import typing as T
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from . import exif_read, utils
+from . import exif_read
 
 
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Adobe": "http://ns.exiftool.org/APP14/Adobe/1.0/",
     "Apple": "http://ns.exiftool.org/MakerNotes/Apple/1.0/",
     "Composite": "http://ns.exiftool.org/Composite/1.0/",
@@ -51,11 +53,11 @@ EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
 
 
 LOG = logging.getLogger(__name__)
+DESCRIPTION_TAG = "rdf:Description"
 _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str)
-_DESCRIPTION_TAG = "rdf:Description"
 
 
-def expand_tag(ns_tag: str, namespaces: T.Dict[str, str]) -> str:
+def expand_tag(ns_tag: str, namespaces: dict[str, str]) -> str:
     try:
         ns, tag = ns_tag.split(":", maxsplit=2)
     except ValueError:
@@ -70,34 +72,23 @@ def canonical_path(path: Path) -> str:
     return str(path.resolve().as_posix())
 
 
-def find_rdf_description_path(element: ET.Element) -> T.Optional[Path]:
+def find_rdf_description_path(element: ET.Element) -> Path | None:
     about = element.get(_EXPANDED_ABOUT_TAG)
     if about is None:
         return None
     return Path(about)
 
 
-def index_rdf_description_by_path(
-    xml_paths: T.Sequence[Path],
-) -> T.Dict[str, ET.Element]:
-    rdf_description_by_path: T.Dict[str, ET.Element] = {}
-
-    for xml_path in utils.find_xml_files(xml_paths):
-        try:
-            etree = ET.parse(xml_path)
-        except ET.ParseError as ex:
-            verbose = LOG.getEffectiveLevel() <= logging.DEBUG
-            if verbose:
-                LOG.warning(f"Failed to parse {xml_path}", exc_info=verbose)
-            else:
-                LOG.warning(f"Failed to parse {xml_path}: {ex}", exc_info=verbose)
-            continue
+def index_rdf_description_by_path_from_xml_element(
+    element: ET.Element,
+) -> dict[str, ET.Element]:
+    rdf_description_by_path: dict[str, ET.Element] = {}
 
-        elements = etree.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
-        for element in elements:
-            path = find_rdf_description_path(element)
-            if path is not None:
-                rdf_description_by_path[canonical_path(path)] = element
+    elements = element.iterfind(DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
+    for element in elements:
+        path = find_rdf_description_path(element)
+        if path is not None:
+            rdf_description_by_path[canonical_path(path)] = element
 
     return rdf_description_by_path
 
@@ -113,7 +104,7 @@ class ExifToolRead(exif_read.ExifReadABC):
     ) -> None:
         self.etree = etree
 
-    def extract_altitude(self) -> T.Optional[float]:
+    def extract_altitude(self) -> float | None:
         """
         Extract altitude
         """
@@ -129,7 +120,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_gps_datetime(
         self, date_tags: T.Sequence[str], time_tags: T.Sequence[str]
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
@@ -143,13 +134,13 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return exif_read.parse_gps_datetime_separately(gpsdate, gpstimestamp)
 
-    def extract_gps_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime(self) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
         return self._extract_gps_datetime(["GPS:GPSDateStamp"], ["GPS:GPSTimeStamp"])
 
-    def extract_gps_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime_from_xmp(self) -> datetime.datetime | None:
         """
         Extract timestamp from XMP GPS field.
         """
@@ -166,7 +157,7 @@ class ExifToolRead(exif_read.ExifReadABC):
         dt_tags: T.Sequence[str],
         subsec_tags: T.Sequence[str],
         offset_tags: T.Sequence[str],
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         dtstr = self._extract_alternative_fields(dt_tags, str)
         if dtstr is None:
             return None
@@ -181,7 +172,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return dt
 
-    def extract_exif_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime_from_xmp(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -220,7 +211,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_exif_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -256,7 +247,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_capture_time(self) -> T.Optional[datetime.datetime]:
+    def extract_capture_time(self) -> datetime.datetime | None:
         """
         Extract capture time from EXIF DateTime tags
         """
@@ -286,7 +277,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_direction(self) -> T.Optional[float]:
+    def extract_direction(self) -> float | None:
         """
         Extract image direction (i.e. compass, heading, bearing)
         """
@@ -299,7 +290,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             float,
         )
 
-    def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]:
+    def extract_lon_lat(self) -> tuple[float, float] | None:
         lon_lat = self._extract_lon_lat("GPS:GPSLongitude", "GPS:GPSLatitude")
         if lon_lat is not None:
             return lon_lat
@@ -318,7 +309,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_lon_lat(
         self, lon_tag: str, lat_tag: str
-    ) -> T.Optional[T.Tuple[float, float]]:
+    ) -> tuple[float, float] | None:
         lon = self._extract_alternative_fields(
             [lon_tag],
             float,
@@ -341,7 +332,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return lon, lat
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         """
         Extract camera make
         """
@@ -360,7 +351,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return make.strip()
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         """
         Extract camera model
         """
@@ -380,7 +371,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return model.strip()
 
-    def extract_width(self) -> T.Optional[int]:
+    def extract_width(self) -> int | None:
         """
         Extract image width in pixels
         """
@@ -395,7 +386,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             int,
         )
 
-    def extract_height(self) -> T.Optional[int]:
+    def extract_height(self) -> int | None:
         """
         Extract image height in pixels
         """
@@ -433,8 +424,8 @@ class ExifToolRead(exif_read.ExifReadABC):
     def _extract_alternative_fields(
         self,
         fields: T.Sequence[str],
-        field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+        field_type: type[_FIELD_TYPE],
+    ) -> _FIELD_TYPE | None:
         for field in fields:
             value = self.etree.findtext(field, namespaces=EXIFTOOL_NAMESPACES)
             if value is None:

mapillary_tools/exiftool_read_video.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import functools
 import logging
@@ -9,7 +11,7 @@ from .telemetry import GPSFix, GPSPoint
 
 
 MAX_TRACK_ID = 10
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Keys": "http://ns.exiftool.org/QuickTime/Keys/1.0/",
     "IFD0": "http://ns.exiftool.org/EXIF/IFD0/1.0/",
     "QuickTime": "http://ns.exiftool.org/QuickTime/QuickTime/1.0/",
@@ -28,7 +30,7 @@ _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str, T.List[str])
 expand_tag = functools.partial(exiftool_read.expand_tag, namespaces=EXIFTOOL_NAMESPACES)
 
 
-def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
+def _maybe_float(text: str | None) -> float | None:
     if text is None:
         return None
     try:
@@ -37,8 +39,8 @@ def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
         return None
 
 
-def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[str]]:
-    texts_by_tag: T.Dict[str, T.List[str]] = {}
+def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> dict[str, list[str]]:
+    texts_by_tag: dict[str, list[str]] = {}
     for element in elements:
         tag = element.tag
         if element.text is not None:
@@ -47,10 +49,10 @@ def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[s
 
 
 def _extract_alternative_fields(
-    texts_by_tag: T.Dict[str, T.List[str]],
+    texts_by_tag: dict[str, list[str]],
     fields: T.Sequence[str],
     field_type: T.Type[_FIELD_TYPE],
-) -> T.Optional[_FIELD_TYPE]:
+) -> _FIELD_TYPE | None:
     for field in fields:
         values = texts_by_tag.get(expand_tag(field))
         if values is None:
@@ -80,15 +82,42 @@ def _extract_alternative_fields(
     return None
 
 
+def _same_gps_point(left: GPSPoint, right: GPSPoint) -> bool:
+    """
+    >>> left =  GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=141.6, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> right = GPSPoint(time=56.0, lat=36.741385, lon=29.021274, alt=142.4, angle=1.54, epoch_time=None, fix=None, precision=None, ground_speed=None)
+    >>> _same_gps_point(left, right)
+    True
+    """
+    return (
+        left.time == right.time
+        and left.lon == right.lon
+        and left.lat == right.lat
+        and left.epoch_time == right.epoch_time
+        and left.angle == right.angle
+    )
+
+
+def _deduplicate_gps_points(
+    track: list[GPSPoint], same_gps_point: T.Callable[[GPSPoint, GPSPoint], bool]
+) -> list[GPSPoint]:
+    deduplicated_track: list[GPSPoint] = []
+    for point in track:
+        if not deduplicated_track or not same_gps_point(deduplicated_track[-1], point):
+            deduplicated_track.append(point)
+    return deduplicated_track
+
+
 def _aggregate_gps_track(
-    texts_by_tag: T.Dict[str, T.List[str]],
-    time_tag: T.Optional[str],
+    texts_by_tag: dict[str, list[str]],
+    time_tag: str | None,
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+) -> list[GPSPoint]:
     """
     Aggregate all GPS data by the tags.
     It requires lat, lon to be present, and their lengths must match.
@@ -139,8 +168,8 @@ def _aggregate_gps_track(
     assert len(timestamps) == expected_length
 
     def _aggregate_float_values_same_length(
-        tag: T.Optional[str],
-    ) -> T.List[T.Optional[float]]:
+        tag: str | None,
+    ) -> list[float | None]:
         if tag is not None:
             vals = [
                 _maybe_float(val)
@@ -161,8 +190,17 @@ def _aggregate_gps_track(
     # aggregate speeds (optional)
     ground_speeds = _aggregate_float_values_same_length(ground_speed_tag)
 
+    # GPS timestamp (optional)
+    epoch_time = None
+    if gps_time_tag is not None:
+        gps_time_text = _extract_alternative_fields(texts_by_tag, [gps_time_tag], str)
+        if gps_time_text is not None:
+            dt = exif_read.parse_gps_datetime(gps_time_text)
+            if dt is not None:
+                epoch_time = geo.as_unix_time(dt)
+
     # build track
-    track = []
+    track: list[GPSPoint] = []
     for timestamp, lon, lat, alt, direction, ground_speed in zip(
         timestamps,
         lons,
@@ -173,22 +211,26 @@ def _aggregate_gps_track(
     ):
         if timestamp is None or lon is None or lat is None:
             continue
-        track.append(
-            GPSPoint(
-                time=timestamp,
-                lon=lon,
-                lat=lat,
-                alt=alt,
-                angle=direction,
-                epoch_time=None,
-                fix=None,
-                precision=None,
-                ground_speed=ground_speed,
-            )
+
+        point = GPSPoint(
+            time=timestamp,
+            lon=lon,
+            lat=lat,
+            alt=alt,
+            angle=direction,
+            epoch_time=epoch_time,
+            fix=None,
+            precision=None,
+            ground_speed=ground_speed,
         )
 
+        if not track or not _same_gps_point(track[-1], point):
+            track.append(point)
+
     track.sort(key=lambda point: point.time)
 
+    track = _deduplicate_gps_points(track, same_gps_point=_same_gps_point)
+
     if time_tag is not None:
         if track:
             first_time = track[0].time
@@ -202,11 +244,11 @@ def _aggregate_samples(
     elements: T.Iterable[ET.Element],
     sample_time_tag: str,
     sample_duration_tag: str,
-) -> T.Generator[T.Tuple[float, float, T.List[ET.Element]], None, None]:
+) -> T.Generator[tuple[float, float, list[ET.Element]], None, None]:
     expanded_sample_time_tag = expand_tag(sample_time_tag)
     expanded_sample_duration_tag = expand_tag(sample_duration_tag)
 
-    accumulated_elements: T.List[ET.Element] = []
+    accumulated_elements: list[ET.Element] = []
     sample_time = None
     sample_duration = None
     for element in elements:
@@ -224,16 +266,17 @@ def _aggregate_samples(
 
 
 def _aggregate_gps_track_by_sample_time(
-    sample_iterator: T.Iterable[T.Tuple[float, float, T.List[ET.Element]]],
+    sample_iterator: T.Iterable[tuple[float, float, list[ET.Element]]],
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-    gps_fix_tag: T.Optional[str] = None,
-    gps_precision_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
-    track: T.List[GPSPoint] = []
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+    gps_fix_tag: str | None = None,
+    gps_precision_tag: str | None = None,
+) -> list[GPSPoint]:
+    track: list[GPSPoint] = []
 
     expanded_gps_fix_tag = None
     if gps_fix_tag is not None:
@@ -297,10 +340,13 @@ class ExifToolReadVideo:
         etree: ET.ElementTree,
     ) -> None:
         self.etree = etree
-        self._texts_by_tag = _index_text_by_tag(self.etree.getroot())
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+        self._texts_by_tag = _index_text_by_tag(root)
         self._all_tags = set(self._texts_by_tag.keys())
 
-    def extract_gps_track(self) -> T.List[geo.Point]:
+    def extract_gps_track(self) -> list[geo.Point]:
         # blackvue and many other cameras
         track_with_fix = self._extract_gps_track_from_quicktime()
         if track_with_fix:
@@ -318,7 +364,7 @@ class ExifToolReadVideo:
 
         return []
 
-    def _extract_make_and_model(self) -> T.Tuple[T.Optional[str], T.Optional[str]]:
+    def _extract_make_and_model(self) -> tuple[str | None, str | None]:
         make = self._extract_alternative_fields(["GoPro:Make"], str)
         model = self._extract_alternative_fields(["GoPro:Model"], str)
         if model is not None:
@@ -349,15 +395,19 @@ class ExifToolReadVideo:
             model = model.strip()
         return make, model
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         make, _ = self._extract_make_and_model()
         return make
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         _, model = self._extract_make_and_model()
         return model
 
-    def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
+    def _extract_gps_track_from_track(self) -> list[GPSPoint]:
+        root = self.etree.getroot()
+        if root is None:
+            raise ValueError("ElementTree root is None")
+
         for track_id in range(1, MAX_TRACK_ID + 1):
             track_ns = f"Track{track_id}"
             if self._all_tags_exists(
@@ -369,7 +419,7 @@ class ExifToolReadVideo:
                 }
             ):
                 sample_iterator = _aggregate_samples(
-                    self.etree.getroot(),
+                    root,
                     f"{track_ns}:SampleTime",
                     f"{track_ns}:SampleDuration",
                 )
@@ -391,15 +441,15 @@ class ExifToolReadVideo:
         self,
         fields: T.Sequence[str],
         field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+    ) -> _FIELD_TYPE | None:
         return _extract_alternative_fields(self._texts_by_tag, fields, field_type)
 
-    def _all_tags_exists(self, tags: T.Set[str]) -> bool:
+    def _all_tags_exists(self, tags: set[str]) -> bool:
         return self._all_tags.issuperset(tags)
 
     def _extract_gps_track_from_quicktime(
         self, namespace: str = "QuickTime"
-    ) -> T.List[GPSPoint]:
+    ) -> list[GPSPoint]:
         if not self._all_tags_exists(
             {
                 expand_tag(f"{namespace}:GPSDateTime"),

mapillary_tools/exiftool_runner.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import subprocess
+import typing as T
+from pathlib import Path
+
+
+class ExiftoolRunner:
+    """
+    Wrapper around ExifTool to run it in a subprocess
+    """
+
+    def __init__(self, exiftool_executable: str = "exiftool", recursive: bool = False):
+        self.exiftool_executable = exiftool_executable
+        self.recursive = recursive
+
+    def _build_args_read_stdin(self) -> list[str]:
+        args: list[str] = [
+            self.exiftool_executable,
+            "-fast",
+            "-q",
+            "-n",  # Disable print conversion
+            "-X",  # XML output
+            "-ee",
+            *["-api", "LargeFileSupport=1"],
+            *["-charset", "filename=utf8"],
+            *["-@", "-"],
+        ]
+
+        if self.recursive:
+            args.append("-r")
+
+        return args
+
+    def extract_xml(self, paths: T.Sequence[Path]) -> str:
+        if not paths:
+            # ExifTool will show its full manual if no files are provided
+            raise ValueError("No files provided to exiftool")
+
+        # To handle non-latin1 filenames under Windows, we pass the path
+        # via stdin. See https://exiftool.org/faq.html#Q18
+        stdin = "\n".join([str(p.resolve()) for p in paths])
+
+        args = self._build_args_read_stdin()
+
+        # Raise FileNotFoundError here if self.exiftool_path not found
+        process = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            input=stdin,
+            encoding="utf-8",
+            # Do not check exit status to allow some files not found
+            # check=True,
+        )
+
+        return process.stdout