mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/exif_write.py

@@ -1,11 +1,11 @@
 # pyre-ignore-all-errors[5, 21, 24]
+from __future__ import annotations
 
 import datetime
 import io
 import json
 import logging
 import math
-import typing as T
 from pathlib import Path
 
 import piexif
@@ -15,9 +15,9 @@ LOG = logging.getLogger(__name__)
 
 
 class ExifEdit:
-    _filename_or_bytes: T.Union[str, bytes]
+    _filename_or_bytes: str | bytes
 
-    def __init__(self, filename_or_bytes: T.Union[Path, bytes]) -> None:
+    def __init__(self, filename_or_bytes: Path | bytes) -> None:
         """Initialize the object"""
         if isinstance(filename_or_bytes, Path):
             # make sure filename is resolved to avoid to be interpretted as bytes in piexif
@@ -25,12 +25,12 @@ class ExifEdit:
             self._filename_or_bytes = str(filename_or_bytes.resolve())
         else:
             self._filename_or_bytes = filename_or_bytes
-        self._ef: T.Dict = piexif.load(self._filename_or_bytes)
+        self._ef: dict = piexif.load(self._filename_or_bytes)
 
     @staticmethod
     def decimal_to_dms(
         value: float, precision: int
-    ) -> T.Tuple[T.Tuple[float, int], T.Tuple[float, int], T.Tuple[float, int]]:
+    ) -> tuple[tuple[float, int], tuple[float, int], tuple[float, int]]:
         """
         Convert decimal position to degrees, minutes, seconds in a fromat supported by EXIF
         """
@@ -40,7 +40,7 @@ class ExifEdit:
 
         return (deg, 1), (min, 1), (sec, precision)
 
-    def add_image_description(self, data: T.Dict) -> None:
+    def add_image_description(self, data: dict) -> None:
         """Add a dict to image description."""
         self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
 
@@ -201,7 +201,7 @@ class ExifEdit:
             piexif.insert(exif_bytes, self._filename_or_bytes, output)
             return output.read()
 
-    def write(self, filename: T.Optional[Path] = None) -> None:
+    def write(self, filename: Path | None = None) -> None:
         """Save exif data to file."""
         if filename is None:
             if not isinstance(self._filename_or_bytes, str):

mapillary_tools/exiftool_read.py

@@ -1,13 +1,15 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import typing as T
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from . import exif_read, utils
+from . import exif_read
 
 
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Adobe": "http://ns.exiftool.org/APP14/Adobe/1.0/",
     "Apple": "http://ns.exiftool.org/MakerNotes/Apple/1.0/",
     "Composite": "http://ns.exiftool.org/Composite/1.0/",
@@ -51,11 +53,11 @@ EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
 
 
 LOG = logging.getLogger(__name__)
+DESCRIPTION_TAG = "rdf:Description"
 _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str)
-_DESCRIPTION_TAG = "rdf:Description"
 
 
-def expand_tag(ns_tag: str, namespaces: T.Dict[str, str]) -> str:
+def expand_tag(ns_tag: str, namespaces: dict[str, str]) -> str:
     try:
         ns, tag = ns_tag.split(":", maxsplit=2)
     except ValueError:
@@ -70,34 +72,23 @@ def canonical_path(path: Path) -> str:
     return str(path.resolve().as_posix())
 
 
-def find_rdf_description_path(element: ET.Element) -> T.Optional[Path]:
+def find_rdf_description_path(element: ET.Element) -> Path | None:
     about = element.get(_EXPANDED_ABOUT_TAG)
     if about is None:
         return None
     return Path(about)
 
 
-def index_rdf_description_by_path(
-    xml_paths: T.Sequence[Path],
-) -> T.Dict[str, ET.Element]:
-    rdf_description_by_path: T.Dict[str, ET.Element] = {}
-
-    for xml_path in utils.find_xml_files(xml_paths):
-        try:
-            etree = ET.parse(xml_path)
-        except ET.ParseError as ex:
-            verbose = LOG.getEffectiveLevel() <= logging.DEBUG
-            if verbose:
-                LOG.warning(f"Failed to parse {xml_path}", exc_info=verbose)
-            else:
-                LOG.warning(f"Failed to parse {xml_path}: {ex}", exc_info=verbose)
-            continue
+def index_rdf_description_by_path_from_xml_element(
+    element: ET.Element,
+) -> dict[str, ET.Element]:
+    rdf_description_by_path: dict[str, ET.Element] = {}
 
-        elements = etree.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
-        for element in elements:
-            path = find_rdf_description_path(element)
-            if path is not None:
-                rdf_description_by_path[canonical_path(path)] = element
+    elements = element.iterfind(DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
+    for element in elements:
+        path = find_rdf_description_path(element)
+        if path is not None:
+            rdf_description_by_path[canonical_path(path)] = element
 
     return rdf_description_by_path
 
@@ -113,7 +104,7 @@ class ExifToolRead(exif_read.ExifReadABC):
     ) -> None:
         self.etree = etree
 
-    def extract_altitude(self) -> T.Optional[float]:
+    def extract_altitude(self) -> float | None:
         """
         Extract altitude
         """
@@ -129,7 +120,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_gps_datetime(
         self, date_tags: T.Sequence[str], time_tags: T.Sequence[str]
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
@@ -143,13 +134,13 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return exif_read.parse_gps_datetime_separately(gpsdate, gpstimestamp)
 
-    def extract_gps_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime(self) -> datetime.datetime | None:
         """
         Extract timestamp from GPS field.
         """
         return self._extract_gps_datetime(["GPS:GPSDateStamp"], ["GPS:GPSTimeStamp"])
 
-    def extract_gps_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_gps_datetime_from_xmp(self) -> datetime.datetime | None:
         """
         Extract timestamp from XMP GPS field.
         """
@@ -166,7 +157,7 @@ class ExifToolRead(exif_read.ExifReadABC):
         dt_tags: T.Sequence[str],
         subsec_tags: T.Sequence[str],
         offset_tags: T.Sequence[str],
-    ) -> T.Optional[datetime.datetime]:
+    ) -> datetime.datetime | None:
         dtstr = self._extract_alternative_fields(dt_tags, str)
         if dtstr is None:
             return None
@@ -181,7 +172,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return dt
 
-    def extract_exif_datetime_from_xmp(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime_from_xmp(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -220,7 +211,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_exif_datetime(self) -> T.Optional[datetime.datetime]:
+    def extract_exif_datetime(self) -> datetime.datetime | None:
         # EXIF DateTimeOriginal: 0x9003 (date/time when original image was taken)
         # EXIF SubSecTimeOriginal: 0x9291 (fractional seconds for DateTimeOriginal)
         # EXIF OffsetTimeOriginal: 0x9011 (time zone for DateTimeOriginal)
@@ -256,7 +247,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_capture_time(self) -> T.Optional[datetime.datetime]:
+    def extract_capture_time(self) -> datetime.datetime | None:
         """
         Extract capture time from EXIF DateTime tags
         """
@@ -286,7 +277,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return None
 
-    def extract_direction(self) -> T.Optional[float]:
+    def extract_direction(self) -> float | None:
         """
         Extract image direction (i.e. compass, heading, bearing)
         """
@@ -299,7 +290,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             float,
         )
 
-    def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]:
+    def extract_lon_lat(self) -> tuple[float, float] | None:
         lon_lat = self._extract_lon_lat("GPS:GPSLongitude", "GPS:GPSLatitude")
         if lon_lat is not None:
             return lon_lat
@@ -318,7 +309,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
     def _extract_lon_lat(
         self, lon_tag: str, lat_tag: str
-    ) -> T.Optional[T.Tuple[float, float]]:
+    ) -> tuple[float, float] | None:
         lon = self._extract_alternative_fields(
             [lon_tag],
             float,
@@ -341,7 +332,7 @@ class ExifToolRead(exif_read.ExifReadABC):
 
         return lon, lat
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         """
         Extract camera make
         """
@@ -360,7 +351,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return make.strip()
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         """
         Extract camera model
         """
@@ -380,7 +371,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             return None
         return model.strip()
 
-    def extract_width(self) -> T.Optional[int]:
+    def extract_width(self) -> int | None:
         """
         Extract image width in pixels
         """
@@ -395,7 +386,7 @@ class ExifToolRead(exif_read.ExifReadABC):
             int,
         )
 
-    def extract_height(self) -> T.Optional[int]:
+    def extract_height(self) -> int | None:
         """
         Extract image height in pixels
         """
@@ -433,8 +424,8 @@ class ExifToolRead(exif_read.ExifReadABC):
     def _extract_alternative_fields(
         self,
         fields: T.Sequence[str],
-        field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+        field_type: type[_FIELD_TYPE],
+    ) -> _FIELD_TYPE | None:
         for field in fields:
             value = self.etree.findtext(field, namespaces=EXIFTOOL_NAMESPACES)
             if value is None:

mapillary_tools/exiftool_read_video.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import functools
 import logging
@@ -9,7 +11,7 @@ from .telemetry import GPSFix, GPSPoint
 
 
 MAX_TRACK_ID = 10
-EXIFTOOL_NAMESPACES: T.Dict[str, str] = {
+EXIFTOOL_NAMESPACES: dict[str, str] = {
     "Keys": "http://ns.exiftool.org/QuickTime/Keys/1.0/",
     "IFD0": "http://ns.exiftool.org/EXIF/IFD0/1.0/",
     "QuickTime": "http://ns.exiftool.org/QuickTime/QuickTime/1.0/",
@@ -28,7 +30,7 @@ _FIELD_TYPE = T.TypeVar("_FIELD_TYPE", int, float, str, T.List[str])
 expand_tag = functools.partial(exiftool_read.expand_tag, namespaces=EXIFTOOL_NAMESPACES)
 
 
-def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
+def _maybe_float(text: str | None) -> float | None:
     if text is None:
         return None
     try:
@@ -37,8 +39,8 @@ def _maybe_float(text: T.Optional[str]) -> T.Optional[float]:
         return None
 
 
-def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[str]]:
-    texts_by_tag: T.Dict[str, T.List[str]] = {}
+def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> dict[str, list[str]]:
+    texts_by_tag: dict[str, list[str]] = {}
     for element in elements:
         tag = element.tag
         if element.text is not None:
@@ -47,10 +49,10 @@ def _index_text_by_tag(elements: T.Iterable[ET.Element]) -> T.Dict[str, T.List[s
 
 
 def _extract_alternative_fields(
-    texts_by_tag: T.Dict[str, T.List[str]],
+    texts_by_tag: dict[str, list[str]],
     fields: T.Sequence[str],
     field_type: T.Type[_FIELD_TYPE],
-) -> T.Optional[_FIELD_TYPE]:
+) -> _FIELD_TYPE | None:
     for field in fields:
         values = texts_by_tag.get(expand_tag(field))
         if values is None:
@@ -81,14 +83,15 @@ def _extract_alternative_fields(
 
 
 def _aggregate_gps_track(
-    texts_by_tag: T.Dict[str, T.List[str]],
-    time_tag: T.Optional[str],
+    texts_by_tag: dict[str, list[str]],
+    time_tag: str | None,
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+) -> list[GPSPoint]:
     """
     Aggregate all GPS data by the tags.
     It requires lat, lon to be present, and their lengths must match.
@@ -139,8 +142,8 @@ def _aggregate_gps_track(
     assert len(timestamps) == expected_length
 
     def _aggregate_float_values_same_length(
-        tag: T.Optional[str],
-    ) -> T.List[T.Optional[float]]:
+        tag: str | None,
+    ) -> list[float | None]:
         if tag is not None:
             vals = [
                 _maybe_float(val)
@@ -161,6 +164,15 @@ def _aggregate_gps_track(
     # aggregate speeds (optional)
     ground_speeds = _aggregate_float_values_same_length(ground_speed_tag)
 
+    # GPS timestamp (optional)
+    epoch_time = None
+    if gps_time_tag is not None:
+        gps_time_text = _extract_alternative_fields(texts_by_tag, [gps_time_tag], str)
+        if gps_time_text is not None:
+            dt = exif_read.parse_gps_datetime(gps_time_text)
+            if dt is not None:
+                epoch_time = geo.as_unix_time(dt)
+
     # build track
     track = []
     for timestamp, lon, lat, alt, direction, ground_speed in zip(
@@ -180,7 +192,7 @@ def _aggregate_gps_track(
                 lat=lat,
                 alt=alt,
                 angle=direction,
-                epoch_time=None,
+                epoch_time=epoch_time,
                 fix=None,
                 precision=None,
                 ground_speed=ground_speed,
@@ -202,11 +214,11 @@ def _aggregate_samples(
     elements: T.Iterable[ET.Element],
     sample_time_tag: str,
     sample_duration_tag: str,
-) -> T.Generator[T.Tuple[float, float, T.List[ET.Element]], None, None]:
+) -> T.Generator[tuple[float, float, list[ET.Element]], None, None]:
     expanded_sample_time_tag = expand_tag(sample_time_tag)
     expanded_sample_duration_tag = expand_tag(sample_duration_tag)
 
-    accumulated_elements: T.List[ET.Element] = []
+    accumulated_elements: list[ET.Element] = []
     sample_time = None
     sample_duration = None
     for element in elements:
@@ -224,16 +236,17 @@ def _aggregate_samples(
 
 
 def _aggregate_gps_track_by_sample_time(
-    sample_iterator: T.Iterable[T.Tuple[float, float, T.List[ET.Element]]],
+    sample_iterator: T.Iterable[tuple[float, float, list[ET.Element]]],
     lon_tag: str,
     lat_tag: str,
-    alt_tag: T.Optional[str] = None,
-    direction_tag: T.Optional[str] = None,
-    ground_speed_tag: T.Optional[str] = None,
-    gps_fix_tag: T.Optional[str] = None,
-    gps_precision_tag: T.Optional[str] = None,
-) -> T.List[GPSPoint]:
-    track: T.List[GPSPoint] = []
+    alt_tag: str | None = None,
+    gps_time_tag: str | None = None,
+    direction_tag: str | None = None,
+    ground_speed_tag: str | None = None,
+    gps_fix_tag: str | None = None,
+    gps_precision_tag: str | None = None,
+) -> list[GPSPoint]:
+    track: list[GPSPoint] = []
 
     expanded_gps_fix_tag = None
     if gps_fix_tag is not None:
@@ -300,7 +313,7 @@ class ExifToolReadVideo:
         self._texts_by_tag = _index_text_by_tag(self.etree.getroot())
         self._all_tags = set(self._texts_by_tag.keys())
 
-    def extract_gps_track(self) -> T.List[geo.Point]:
+    def extract_gps_track(self) -> list[geo.Point]:
         # blackvue and many other cameras
         track_with_fix = self._extract_gps_track_from_quicktime()
         if track_with_fix:
@@ -318,7 +331,7 @@ class ExifToolReadVideo:
 
         return []
 
-    def _extract_make_and_model(self) -> T.Tuple[T.Optional[str], T.Optional[str]]:
+    def _extract_make_and_model(self) -> tuple[str | None, str | None]:
         make = self._extract_alternative_fields(["GoPro:Make"], str)
         model = self._extract_alternative_fields(["GoPro:Model"], str)
         if model is not None:
@@ -349,15 +362,15 @@ class ExifToolReadVideo:
             model = model.strip()
         return make, model
 
-    def extract_make(self) -> T.Optional[str]:
+    def extract_make(self) -> str | None:
         make, _ = self._extract_make_and_model()
         return make
 
-    def extract_model(self) -> T.Optional[str]:
+    def extract_model(self) -> str | None:
         _, model = self._extract_make_and_model()
         return model
 
-    def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
+    def _extract_gps_track_from_track(self) -> list[GPSPoint]:
         for track_id in range(1, MAX_TRACK_ID + 1):
             track_ns = f"Track{track_id}"
             if self._all_tags_exists(
@@ -391,15 +404,15 @@ class ExifToolReadVideo:
         self,
         fields: T.Sequence[str],
         field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+    ) -> _FIELD_TYPE | None:
         return _extract_alternative_fields(self._texts_by_tag, fields, field_type)
 
-    def _all_tags_exists(self, tags: T.Set[str]) -> bool:
+    def _all_tags_exists(self, tags: set[str]) -> bool:
         return self._all_tags.issuperset(tags)
 
     def _extract_gps_track_from_quicktime(
         self, namespace: str = "QuickTime"
-    ) -> T.List[GPSPoint]:
+    ) -> list[GPSPoint]:
         if not self._all_tags_exists(
             {
                 expand_tag(f"{namespace}:GPSDateTime"),

mapillary_tools/exiftool_runner.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import platform
+import shutil
+import subprocess
+import typing as T
+from pathlib import Path
+
+
+class ExiftoolRunner:
+    """
+    Wrapper around ExifTool to run it in a subprocess
+    """
+
+    def __init__(self, exiftool_path: str | None = None, recursive: bool = False):
+        if exiftool_path is None:
+            exiftool_path = self._search_preferred_exiftool_path()
+        self.exiftool_path = exiftool_path
+        self.recursive = recursive
+
+    def _search_preferred_exiftool_path(self) -> str:
+        system = platform.system()
+
+        if system and system.lower() == "windows":
+            exiftool_paths = ["exiftool.exe", "exiftool"]
+        else:
+            exiftool_paths = ["exiftool", "exiftool.exe"]
+
+        for path in exiftool_paths:
+            full_path = shutil.which(path)
+            if full_path:
+                return path
+
+        # Always return the prefered one, even if it is not found,
+        # and let the subprocess.run figure out the error later
+        return exiftool_paths[0]
+
+    def _build_args_read_stdin(self) -> list[str]:
+        args: list[str] = [
+            self.exiftool_path,
+            "-q",
+            "-n",  # Disable print conversion
+            "-X",  # XML output
+            "-ee",
+            *["-api", "LargeFileSupport=1"],
+            *["-charset", "filename=utf8"],
+            *["-@", "-"],
+        ]
+
+        if self.recursive:
+            args.append("-r")
+
+        return args
+
+    def extract_xml(self, paths: T.Sequence[Path]) -> str:
+        if not paths:
+            # ExifTool will show its full manual if no files are provided
+            raise ValueError("No files provided to exiftool")
+
+        # To handle non-latin1 filenames under Windows, we pass the path
+        # via stdin. See https://exiftool.org/faq.html#Q18
+        stdin = "\n".join([str(p.resolve()) for p in paths])
+
+        args = self._build_args_read_stdin()
+
+        # Raise FileNotFoundError here if self.exiftool_path not found
+        process = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            input=stdin,
+            encoding="utf-8",
+            # Do not check exit status to allow some files not found
+            # check=True,
+        )
+
+        return process.stdout