mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a1__py3-none-any.whl

mapillary_tools/exif_read.py

@@ -1,7 +1,11 @@
+from __future__ import annotations
+
 import abc
 import datetime
+import io
 import logging
 import re
+import struct
 import typing as T
 import xml.etree.ElementTree as et
 from fractions import Fraction
@@ -11,6 +15,7 @@ import exifread
 from exifread.utils import Ratio
 
 
+LOG = logging.getLogger(__name__)
 XMP_NAMESPACES = {
     "exif": "http://ns.adobe.com/exif/1.0/",
     "tiff": "http://ns.adobe.com/tiff/1.0/",
@@ -474,9 +479,9 @@ class ExifReadFromXMP(ExifReadABC):
 
     def _extract_alternative_fields(
         self,
-        fields: T.Sequence[str],
-        field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+        fields: T.Iterable[str],
+        field_type: type[_FIELD_TYPE],
+    ) -> _FIELD_TYPE | None:
         """
         Extract a value for a list of ordered fields.
         Return the value of the first existed field in the list
@@ -508,6 +513,86 @@ class ExifReadFromXMP(ExifReadABC):
         return None
 
 
+def extract_xmp_efficiently(fp) -> T.Optional[str]:
+    """
+    Extract XMP metadata from a JPEG file efficiently by reading only necessary chunks.
+
+    Args:
+        image_path (str): Path to the JPEG image file
+
+    Returns:
+        str: Formatted XML string containing XMP metadata, or None if no XMP data found
+    """
+    # JPEG markers
+    SOI_MARKER = b"\xff\xd8"  # Start of Image
+    APP1_MARKER = b"\xff\xe1"  # Application Segment 1 (where XMP usually lives)
+    XMP_IDENTIFIER = b"http://ns.adobe.com/xap/1.0/\x00"
+    XMP_META_TAG_BEGIN = b"<x:xmpmeta"
+    XMP_META_TAG_END = b"</x:xmpmeta>"
+
+    # Check for JPEG signature (SOI marker)
+    if fp.read(2) != SOI_MARKER:
+        return None
+
+    while True:
+        # Read marker
+        marker_bytes = fp.read(2)
+        if len(marker_bytes) < 2:
+            # End of file
+            break
+
+        # If not APP1, skip this segment
+        if marker_bytes != APP1_MARKER:
+            # Read length field (includes the length bytes themselves)
+            length_bytes = fp.read(2)
+            if len(length_bytes) < 2:
+                break
+
+            length = struct.unpack(">H", length_bytes)[0]
+            # Skip the rest of this segment (-2 because length includes length bytes)
+            fp.seek(length - 2, io.SEEK_CUR)
+            continue
+
+        # It's an APP1 segment - read length
+        length_bytes = fp.read(2)
+        if len(length_bytes) < 2:
+            break
+
+        length = struct.unpack(">H", length_bytes)[0]
+        segment_data_length = length - 2  # Subtract length field size
+
+        # Read enough bytes to check for XMP identifier
+        identifier_check = fp.read(len(XMP_IDENTIFIER))
+        if len(identifier_check) < len(XMP_IDENTIFIER):
+            break
+
+        # Check if this APP1 contains XMP data
+        if identifier_check == XMP_IDENTIFIER:
+            # We found XMP data - read the rest of the segment
+            remaining_length = segment_data_length - len(XMP_IDENTIFIER)
+            if remaining_length > 128 * 1024 * 1024:
+                raise ValueError("XMP data too large")
+            xmp_data = fp.read(remaining_length)
+
+            # Process the XMP data
+            begin_idx = xmp_data.find(XMP_META_TAG_BEGIN)
+            if begin_idx >= 0:
+                end_idx = xmp_data.rfind(XMP_META_TAG_END, begin_idx)
+                if end_idx >= 0:
+                    xmp_data = xmp_data[begin_idx : end_idx + len(XMP_META_TAG_END)]
+                else:
+                    xmp_data = xmp_data[begin_idx:]
+
+            return xmp_data.decode("utf-8")
+        else:
+            # Not XMP data - skip the rest of this APP1 segment
+            # We already read the identifier_check bytes, so subtract that
+            fp.seek(segment_data_length - len(identifier_check), io.SEEK_CUR)
+
+    # If we reach here, no XMP data was found
+    return None
+
+
 class ExifReadFromEXIF(ExifReadABC):
     """
     EXIF class for reading exif from an image
@@ -520,16 +605,20 @@ class ExifReadFromEXIF(ExifReadABC):
         if isinstance(path_or_stream, Path):
             with path_or_stream.open("rb") as fp:
                 try:
-                    self.tags = exifread.process_file(fp, details=True, debug=True)
-                except Exception:
+                    # Turn off details and debug for performance reasons
+                    self.tags = exifread.process_file(fp, details=False, debug=False)
+                except Exception as ex:
+                    LOG.warning("Error reading EXIF from %s: %s", path_or_stream, ex)
                     self.tags = {}
 
         else:
             try:
+                # Turn off details and debug for performance reasons
                 self.tags = exifread.process_file(
-                    path_or_stream, details=True, debug=True
+                    path_or_stream, details=False, debug=False
                 )
-            except Exception:
+            except Exception as ex:
+                LOG.warning("Error reading EXIF: %s", ex)
                 self.tags = {}
 
     def extract_altitude(self) -> T.Optional[float]:
@@ -649,17 +738,7 @@ class ExifReadFromEXIF(ExifReadABC):
             "GPS GPSImgDirection",
             "GPS GPSTrack",
         ]
-        direction = self._extract_alternative_fields(fields, float)
-        if direction is not None:
-            if direction > 360:
-                # fix negative value wrongly parsed in exifread
-                # -360 degree -> 4294966935 when converting from hex
-                bearing1 = bin(int(direction))[2:]
-                bearing2 = "".join([str(int(int(a) == 0)) for a in bearing1])
-                direction = -float(int(bearing2, 2))
-            direction %= 360
-
-        return direction
+        return self._extract_alternative_fields(fields, float)
 
     def extract_lon_lat(self) -> T.Optional[T.Tuple[float, float]]:
         lat_tag = self.tags.get("GPS GPSLatitude")
@@ -687,7 +766,9 @@ class ExifReadFromEXIF(ExifReadABC):
         """
         Extract camera make
         """
-        make = self._extract_alternative_fields(["Image Make", "EXIF LensMake"], str)
+        make = self._extract_alternative_fields(
+            ["Image Make", "EXIF Make", "EXIF LensMake"], str
+        )
         if make is None:
             return None
         return make.strip()
@@ -696,7 +777,9 @@ class ExifReadFromEXIF(ExifReadABC):
         """
         Extract camera model
         """
-        model = self._extract_alternative_fields(["Image Model", "EXIF LensModel"], str)
+        model = self._extract_alternative_fields(
+            ["Image Model", "EXIF Model", "EXIF LensModel"], str
+        )
         if model is None:
             return None
         return model.strip()
@@ -731,8 +814,8 @@ class ExifReadFromEXIF(ExifReadABC):
     def _extract_alternative_fields(
         self,
         fields: T.Sequence[str],
-        field_type: T.Type[_FIELD_TYPE],
-    ) -> T.Optional[_FIELD_TYPE]:
+        field_type: type[_FIELD_TYPE],
+    ) -> _FIELD_TYPE | None:
         """
         Extract a value for a list of ordered fields.
         Return the value of the first existed field in the list
@@ -775,27 +858,54 @@ class ExifReadFromEXIF(ExifReadABC):
 
 
 class ExifRead(ExifReadFromEXIF):
+    """
+    Extract properties from EXIF first and then XMP
+    NOTE: For performance reasons, XMP is only extracted if EXIF does not contain the required fields
+    """
+
     def __init__(self, path_or_stream: T.Union[Path, T.BinaryIO]) -> None:
         super().__init__(path_or_stream)
-        self._xmp = self._extract_xmp()
+        self._path_or_stream = path_or_stream
+        self._xml_extracted: bool = False
+        self._cached_xml: T.Optional[ExifReadFromXMP] = None
+
+    def _xmp_with_reason(self, reason: str) -> T.Optional[ExifReadFromXMP]:
+        if not self._xml_extracted:
+            LOG.debug('Extracting XMP for "%s"', reason)
+            self._cached_xml = self._extract_xmp()
+            self._xml_extracted = True
+
+        return self._cached_xml
 
     def _extract_xmp(self) -> T.Optional[ExifReadFromXMP]:
-        application_notes = self.extract_application_notes()
-        if application_notes is None:
-            return None
+        xml_str = self.extract_application_notes()
+        if xml_str is None:
+            if isinstance(self._path_or_stream, Path):
+                with self._path_or_stream.open("rb") as fp:
+                    xml_str = extract_xmp_efficiently(fp)
+            else:
+                self._path_or_stream.seek(0, io.SEEK_SET)
+                xml_str = extract_xmp_efficiently(self._path_or_stream)
+
+            if xml_str is None:
+                return None
+
         try:
-            e = et.fromstring(application_notes)
-        except et.ParseError:
+            e = et.fromstring(xml_str)
+        except et.ParseError as ex:
+            LOG.warning("Error parsing XMP XML: %s: %s", ex, xml_str)
             return None
+
         return ExifReadFromXMP(et.ElementTree(e))
 
     def extract_altitude(self) -> T.Optional[float]:
         val = super().extract_altitude()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("altitude")
+        if xmp is None:
             return None
-        val = self._xmp.extract_altitude()
+        val = xmp.extract_altitude()
         if val is not None:
             return val
         return None
@@ -804,20 +914,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_capture_time()
         if val is not None:
             return val
-        if self._xmp is None:
-            return None
-        val = self._xmp.extract_capture_time()
-        if val is not None:
-            return val
-        return None
-
-    def extract_direction(self) -> T.Optional[float]:
-        val = super().extract_direction()
-        if val is not None:
-            return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("capture_time")
+        if xmp is None:
             return None
-        val = self._xmp.extract_direction()
+        val = xmp.extract_capture_time()
         if val is not None:
             return val
         return None
@@ -826,9 +926,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_lon_lat()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("lon_lat")
+        if xmp is None:
             return None
-        val = self._xmp.extract_lon_lat()
+        val = xmp.extract_lon_lat()
         if val is not None:
             return val
         return None
@@ -837,9 +938,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_make()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("make")
+        if xmp is None:
             return None
-        val = self._xmp.extract_make()
+        val = xmp.extract_make()
         if val is not None:
             return val
         return None
@@ -848,9 +950,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_model()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("model")
+        if xmp is None:
             return None
-        val = self._xmp.extract_model()
+        val = xmp.extract_model()
         if val is not None:
             return val
         return None
@@ -859,9 +962,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_width()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("width")
+        if xmp is None:
             return None
-        val = self._xmp.extract_width()
+        val = xmp.extract_width()
         if val is not None:
             return val
         return None
@@ -870,9 +974,10 @@ class ExifRead(ExifReadFromEXIF):
         val = super().extract_height()
         if val is not None:
             return val
-        if self._xmp is None:
+        xmp = self._xmp_with_reason("width")
+        if xmp is None:
             return None
-        val = self._xmp.extract_height()
+        val = xmp.extract_height()
         if val is not None:
             return val
         return None

mapillary_tools/exiftool_read.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import typing as T
@@ -93,11 +95,23 @@ def index_rdf_description_by_path(
                 LOG.warning(f"Failed to parse {xml_path}: {ex}", exc_info=verbose)
             continue
 
-        elements = etree.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
-        for element in elements:
-            path = find_rdf_description_path(element)
-            if path is not None:
-                rdf_description_by_path[canonical_path(path)] = element
+        rdf_description_by_path.update(
+            index_rdf_description_by_path_from_xml_element(etree.getroot())
+        )
+
+    return rdf_description_by_path
+
+
+def index_rdf_description_by_path_from_xml_element(
+    element: ET.Element,
+) -> dict[str, ET.Element]:
+    rdf_description_by_path: dict[str, ET.Element] = {}
+
+    elements = element.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES)
+    for element in elements:
+        path = find_rdf_description_path(element)
+        if path is not None:
+            rdf_description_by_path[canonical_path(path)] = element
 
     return rdf_description_by_path
 

mapillary_tools/exiftool_read_video.py

@@ -86,6 +86,7 @@ def _aggregate_gps_track(
     lon_tag: str,
     lat_tag: str,
     alt_tag: T.Optional[str] = None,
+    gps_time_tag: T.Optional[str] = None,
     direction_tag: T.Optional[str] = None,
     ground_speed_tag: T.Optional[str] = None,
 ) -> T.List[GPSPoint]:
@@ -161,6 +162,15 @@ def _aggregate_gps_track(
     # aggregate speeds (optional)
     ground_speeds = _aggregate_float_values_same_length(ground_speed_tag)
 
+    # GPS timestamp (optional)
+    epoch_time = None
+    if gps_time_tag is not None:
+        gps_time_text = _extract_alternative_fields(texts_by_tag, [gps_time_tag], str)
+        if gps_time_text is not None:
+            dt = exif_read.parse_gps_datetime(gps_time_text)
+            if dt is not None:
+                epoch_time = geo.as_unix_time(dt)
+
     # build track
     track = []
     for timestamp, lon, lat, alt, direction, ground_speed in zip(
@@ -180,7 +190,7 @@ def _aggregate_gps_track(
                 lat=lat,
                 alt=alt,
                 angle=direction,
-                epoch_time=None,
+                epoch_time=epoch_time,
                 fix=None,
                 precision=None,
                 ground_speed=ground_speed,
@@ -228,6 +238,7 @@ def _aggregate_gps_track_by_sample_time(
     lon_tag: str,
     lat_tag: str,
     alt_tag: T.Optional[str] = None,
+    gps_time_tag: T.Optional[str] = None,
     direction_tag: T.Optional[str] = None,
     ground_speed_tag: T.Optional[str] = None,
     gps_fix_tag: T.Optional[str] = None,

mapillary_tools/exiftool_runner.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import platform
+import shutil
+import subprocess
+import typing as T
+from pathlib import Path
+
+
+class ExiftoolRunner:
+    """
+    Wrapper around ExifTool to run it in a subprocess
+    """
+
+    def __init__(self, exiftool_path: str | None = None, recursive: bool = False):
+        if exiftool_path is None:
+            exiftool_path = self._search_preferred_exiftool_path()
+        self.exiftool_path = exiftool_path
+        self.recursive = recursive
+
+    def _search_preferred_exiftool_path(self) -> str:
+        system = platform.system()
+
+        if system and system.lower() == "windows":
+            exiftool_paths = ["exiftool.exe", "exiftool"]
+        else:
+            exiftool_paths = ["exiftool", "exiftool.exe"]
+
+        for path in exiftool_paths:
+            full_path = shutil.which(path)
+            if full_path:
+                return path
+
+        # Always return the prefered one, even if it is not found,
+        # and let the subprocess.run figure out the error later
+        return exiftool_paths[0]
+
+    def _build_args_read_stdin(self) -> list[str]:
+        args: list[str] = [
+            self.exiftool_path,
+            "-q",
+            "-n",  # Disable print conversion
+            "-X",  # XML output
+            "-ee",
+            *["-api", "LargeFileSupport=1"],
+            *["-charset", "filename=utf8"],
+            *["-@", "-"],
+        ]
+
+        if self.recursive:
+            args.append("-r")
+
+        return args
+
+    def extract_xml(self, paths: T.Sequence[Path]) -> str:
+        if not paths:
+            # ExifTool will show its full manual if no files are provided
+            raise ValueError("No files provided to exiftool")
+
+        # To handle non-latin1 filenames under Windows, we pass the path
+        # via stdin. See https://exiftool.org/faq.html#Q18
+        stdin = "\n".join([str(p.resolve()) for p in paths])
+
+        args = self._build_args_read_stdin()
+
+        # Raise FileNotFoundError here if self.exiftool_path not found
+        process = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            input=stdin,
+            encoding="utf-8",
+            # Do not check exit status to allow some files not found
+            # check=True,
+        )
+
+        return process.stdout