mapillary-tools 0.14.0b1__py3-none-any.whl → 0.14.2__py3-none-any.whl

mapillary_tools/commands/upload.py

@@ -8,13 +8,13 @@ from .process import bold_text
 
 class Command:
     name = "upload"
-    help = "upload images to Mapillary"
+    help = "Upload processed data to Mapillary"
 
     @staticmethod
     def add_common_upload_options(group):
         group.add_argument(
             "--user_name",
-            help="The Mapillary user account to upload to. If you only have one account authorized, it will upload to that account by default.",
+            help="The Mapillary user account to upload to.",
             required=False,
         )
         group.add_argument(
@@ -23,9 +23,38 @@ class Command:
             default=None,
             required=False,
         )
+        group.add_argument(
+            "--num_upload_workers",
+            help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
+            default=constants.MAX_IMAGE_UPLOAD_WORKERS,
+            type=int,
+            required=False,
+        )
+        group.add_argument(
+            "--reupload",
+            help="Re-upload data that has already been uploaded.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
         group.add_argument(
             "--dry_run",
-            help='Instead of uploading to the Mapillary server, simulate uploading to the local directory "mapillary_public_uploads" for debugging purposes.',
+            "--dryrun",
+            help="[DEVELOPMENT] Simulate upload by sending data to a local directory instead of Mapillary servers. Uses a temporary directory by default unless specified by MAPILLARY_UPLOAD_ENDPOINT environment variable.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--nofinish",
+            help="[DEVELOPMENT] Upload data without finalizing. The data will NOT be stored permanently or appear on the Mapillary website.",
+            action="store_true",
+            default=False,
+            required=False,
+        )
+        group.add_argument(
+            "--noresume",
+            help="[DEVELOPMENT] Start upload from the beginning, ignoring any previously interrupted upload sessions.",
             action="store_true",
             default=False,
             required=False,
@@ -35,7 +64,7 @@ class Command:
         group = parser.add_argument_group(bold_text("UPLOAD OPTIONS"))
         group.add_argument(
             "--desc_path",
-            help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
+            help=f'Path to the description file with processed image and video metadata (from process command). Use "-" for STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
             default=None,
             required=False,
         )

mapillary_tools/config.py

@@ -6,6 +6,8 @@ import sys
 import typing as T
 from typing import TypedDict
 
+import jsonschema
+
 if sys.version_info >= (3, 11):
     from typing import Required
 else:
@@ -50,6 +52,9 @@ UserItemSchema = {
 }
 
 
+UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
+
+
 def _load_config(config_path: str) -> configparser.ConfigParser:
     config = configparser.ConfigParser()
     # Override to not change option names (by default it will lower them)

mapillary_tools/constants.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import functools
 import os
+import tempfile
 
 import appdirs
 
@@ -8,44 +10,92 @@ _ENV_PREFIX = "MAPILLARY_TOOLS_"
 
 
 def _yes_or_no(val: str) -> bool:
-    return val.strip().upper() in [
-        "1",
-        "TRUE",
-        "YES",
-    ]
+    return val.strip().upper() in ["1", "TRUE", "YES"]
 
 
-# In meters
-CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+def _parse_scaled_integers(
+    value: str, scale: dict[str, int] | None = None
+) -> int | None:
+    """
+    >>> scale = {"": 1, "b": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024}
+    >>> _parse_scaled_integers("0", scale=scale)
+    0
+    >>> _parse_scaled_integers("10", scale=scale)
+    10
+    >>> _parse_scaled_integers("100B", scale=scale)
+    100
+    >>> _parse_scaled_integers("100k", scale=scale)
+    102400
+    >>> _parse_scaled_integers("100t", scale=scale)
+    Traceback (most recent call last):
+    ValueError: Expect valid integer ends with , b, K, M, G, but got 100T
+    """
+
+    if scale is None:
+        scale = {"": 1}
+
+    value = value.strip().upper()
+
+    if value in ["INF", "INFINITY"]:
+        return None
+
+    try:
+        for k, v in scale.items():
+            k = k.upper()
+            if k and value.endswith(k):
+                return int(value[: -len(k)]) * v
+
+        if "" in scale:
+            return int(value) * scale[""]
+    except ValueError:
+        pass
+
+    raise ValueError(
+        f"Expect valid integer ends with {', '.join(scale.keys())}, but got {value}"
+    )
+
+
+_parse_pixels = functools.partial(
+    _parse_scaled_integers,
+    scale={
+        "": 1,
+        "K": 1000,
+        "M": 1000 * 1000,
+        "MP": 1000 * 1000,
+        "G": 1000 * 1000 * 1000,
+        "GP": 1000 * 1000 * 1000,
+    },
+)
+
+_parse_filesize = functools.partial(
+    _parse_scaled_integers,
+    scale={"B": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024},
+)
+
+###################
+##### GENERAL #####
+###################
+USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
+PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
+
+
+############################
+##### VIDEO PROCESSING #####
+############################
 # In seconds
-CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
-DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
-DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
-MAX_AVG_SPEED = float(
-    os.getenv(_ENV_PREFIX + "MAX_AVG_SPEED", 400_000 / 3600)
-)  # 400 KM/h
-# in seconds
 VIDEO_SAMPLE_INTERVAL = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_INTERVAL", -1))
-# in meters
+# In meters
 VIDEO_SAMPLE_DISTANCE = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_DISTANCE", 3))
 VIDEO_DURATION_RATIO = float(os.getenv(_ENV_PREFIX + "VIDEO_DURATION_RATIO", 1))
 FFPROBE_PATH: str = os.getenv(_ENV_PREFIX + "FFPROBE_PATH", "ffprobe")
 FFMPEG_PATH: str = os.getenv(_ENV_PREFIX + "FFMPEG_PATH", "ffmpeg")
-# When not set, MT will try to check both "exiftool" and "exiftool.exe" from $PATH
-EXIFTOOL_PATH: str | None = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH")
+EXIFTOOL_PATH: str = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH", "exiftool")
 IMAGE_DESCRIPTION_FILENAME = os.getenv(
     _ENV_PREFIX + "IMAGE_DESCRIPTION_FILENAME", "mapillary_image_description.json"
 )
 SAMPLED_VIDEO_FRAMES_FILENAME = os.getenv(
     _ENV_PREFIX + "SAMPLED_VIDEO_FRAMES_FILENAME", "mapillary_sampled_video_frames"
 )
-USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
-# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
-# for uploading data to MLY upload service.
-# Changing this size does not change the number of requests nor affect upload performance,
-# but it affects the responsiveness of the upload progress bar
-UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
-
 # DoP value, the lower the better
 # See https://github.com/gopro/gpmf-parser#hero5-black-with-gps-enabled-adds
 # It is used to filter out noisy points
@@ -54,40 +104,72 @@ GOPRO_MAX_DOP100 = int(os.getenv(_ENV_PREFIX + "GOPRO_MAX_DOP100", 1000))
 GOPRO_GPS_FIXES: set[int] = set(
     int(fix) for fix in os.getenv(_ENV_PREFIX + "GOPRO_GPS_FIXES", "2,3").split(",")
 )
-MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
-
 # GPS precision, in meters, is used to filter outliers
 GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
+MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
+    os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
+)
 
+
+#################################
+###### SEQUENCE PROCESSING ######
+#################################
+# In meters
+CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
+# In seconds
+CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
+DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
+DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
+MAX_CAPTURE_SPEED_KMH = float(
+    os.getenv(_ENV_PREFIX + "MAX_CAPTURE_SPEED_KMH", 400)
+)  # 400 KM/h
 # WARNING: Changing the following envvars might result in failed uploads
 # Max number of images per sequence
-MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
+MAX_SEQUENCE_LENGTH: int | None = _parse_scaled_integers(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", "1000")
+)
 # Max file size per sequence (sum of image filesizes in the sequence)
-MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
+MAX_SEQUENCE_FILESIZE: int | None = _parse_filesize(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
+)
 # Max number of pixels per sequence (sum of image pixels in the sequence)
-MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
-
-PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
-
-_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
-    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
+MAX_SEQUENCE_PIXELS: int | None = _parse_pixels(
+    os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
 )
 
+
+##################
+##### UPLOAD #####
+##################
 MAPILLARY_DISABLE_API_LOGGING: bool = _yes_or_no(
     os.getenv("MAPILLARY_DISABLE_API_LOGGING", "NO")
 )
+MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
+    "MAPILLARY_UPLOAD_HISTORY_PATH", os.path.join(USER_DATA_DIR, "upload_history")
+)
+UPLOAD_CACHE_DIR: str = os.getenv(
+    _ENV_PREFIX + "UPLOAD_CACHE_DIR",
+    os.path.join(tempfile.gettempdir(), "mapillary_tools", "upload_cache"),
+)
+# The minimal upload speed is used to calculate the read timeout to avoid upload hanging:
+# timeout = upload_size / MIN_UPLOAD_SPEED
+MIN_UPLOAD_SPEED: int | None = _parse_filesize(
+    os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K")  # 50 Kb/s
+)
+# Maximum number of parallel workers for uploading images within a single sequence.
+# NOTE: Sequences themselves are uploaded sequentially, not in parallel.
+MAX_IMAGE_UPLOAD_WORKERS: int = int(
+    os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 4)
+)
+# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
+# for uploading data to MLY upload service.
+# Changing this size does not change the number of requests nor affect upload performance,
+# but it affects the responsiveness of the upload progress bar
+UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 2))
+MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
 MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
     os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
 )
-MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
-    os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
-)
-MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
-    "MAPILLARY_UPLOAD_HISTORY_PATH",
-    os.path.join(
-        USER_DATA_DIR,
-        "upload_history",
-    ),
+_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
+    os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
 )
-
-MAX_IMAGE_UPLOAD_WORKERS = int(os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 64))

mapillary_tools/exceptions.py

@@ -51,6 +51,10 @@ class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError):
     pass
 
 
+class MapillaryInvalidVideoError(MapillaryDescriptionError):
+    pass
+
+
 class MapillaryGPXEmptyError(MapillaryDescriptionError):
     pass
 

mapillary_tools/exif_read.py

@@ -871,7 +871,8 @@ class ExifRead(ExifReadFromEXIF):
 
     def _xmp_with_reason(self, reason: str) -> ExifReadFromXMP | None:
         if not self._xml_extracted:
-            LOG.debug('Extracting XMP for "%s"', reason)
+            # TODO Disabled because too verbose but still useful to know
+            # LOG.debug('Extracting XMP for "%s"', reason)
             self._cached_xml = self._extract_xmp()
             self._xml_extracted = True
 

mapillary_tools/exif_write.py

@@ -42,7 +42,9 @@ class ExifEdit:
 
     def add_image_description(self, data: dict) -> None:
         """Add a dict to image description."""
-        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
+        self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(
+            data, sort_keys=True, separators=(",", ":")
+        )
 
     def add_orientation(self, orientation: int) -> None:
         """Add image orientation to image."""

mapillary_tools/geo.py

@@ -51,6 +51,22 @@ def gps_distance(latlon_1: tuple[float, float], latlon_2: tuple[float, float]) -
     return math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2 + (z1 - z2) ** 2)
 
 
+def avg_speed(sequence: T.Sequence[PointLike]) -> float:
+    total_distance = 0.0
+    for cur, nxt in pairwise(sequence):
+        total_distance += gps_distance((cur.lat, cur.lon), (nxt.lat, nxt.lon))
+
+    if sequence:
+        time_diff = sequence[-1].time - sequence[0].time
+    else:
+        time_diff = 0.0
+
+    if time_diff == 0.0:
+        return float("inf")
+
+    return total_distance / time_diff
+
+
 def compute_bearing(
     latlon_1: tuple[float, float],
     latlon_2: tuple[float, float],

mapillary_tools/geotag/base.py

@@ -46,7 +46,7 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
                 map_results,
                 desc="Extracting images",
                 unit="images",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
+                disable=LOG.isEnabledFor(logging.DEBUG),
                 total=len(extractors),
             )
         )
@@ -62,6 +62,8 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
         try:
             return extractor.extract()
         except exceptions.MapillaryDescriptionError as ex:
+            if LOG.isEnabledFor(logging.DEBUG):
+                LOG.error(f"{cls.__name__}({image_path.name}): {ex}")
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
@@ -112,7 +114,7 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
                 map_results,
                 desc="Extracting videos",
                 unit="videos",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
+                disable=LOG.isEnabledFor(logging.DEBUG),
                 total=len(extractors),
             )
         )
@@ -128,6 +130,8 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
         try:
             return extractor.extract()
         except exceptions.MapillaryDescriptionError as ex:
+            if LOG.isEnabledFor(logging.DEBUG):
+                LOG.error(f"{cls.__name__}({video_path.name}): {ex}")
             return types.describe_error_metadata(
                 ex, video_path, filetype=types.FileType.VIDEO
             )

mapillary_tools/geotag/factory.py

@@ -67,7 +67,14 @@ def process(
     reprocessable_paths = set(paths)
 
     for idx, option in enumerate(options):
-        LOG.debug("Processing %d files with %s", len(reprocessable_paths), option)
+        if LOG.isEnabledFor(logging.DEBUG):
+            LOG.info(
+                f"==> Processing {len(reprocessable_paths)} files with source {option}..."
+            )
+        else:
+            LOG.info(
+                f"==> Processing {len(reprocessable_paths)} files with source {option.source.value}..."
+            )
 
         image_videos, video_paths = _filter_images_and_videos(
             reprocessable_paths, option.filetypes
@@ -114,6 +121,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
                 exceptions.MapillaryGeoTaggingError,
                 exceptions.MapillaryVideoGPSNotFoundError,
                 exceptions.MapillaryExiftoolNotFoundError,
+                exceptions.MapillaryExifToolXMLNotFoundError,
             ),
         ):
             return True

mapillary_tools/geotag/geotag_images_from_exiftool.py

@@ -93,7 +93,7 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
             LOG.warning(
                 "Failed to parse ExifTool XML: %s",
                 str(ex),
-                exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             rdf_by_path = {}
         else:

mapillary_tools/geotag/geotag_images_from_gpx.py

@@ -20,12 +20,6 @@ from .geotag_images_from_exif import ImageEXIFExtractor
 LOG = logging.getLogger(__name__)
 
 
-class SyncMode:
-    SYNC = "sync"
-    STRICT_SYNC = "strict_sync"
-    RESET = "reset"
-
-
 class GeotagImagesFromGPX(GeotagImagesFromGeneric):
     def __init__(
         self,

mapillary_tools/geotag/geotag_videos_from_exiftool.py

@@ -61,21 +61,42 @@ class GeotagVideosFromExifToolXML(GeotagVideosFromGeneric):
     def find_rdf_by_path(
         cls, option: options.SourcePathOption, paths: T.Iterable[Path]
     ) -> dict[str, ET.Element]:
+        # Find RDF descriptions by path in RDF description
+        # Sources are matched based on the paths in "rdf:about" in XML elements
+        # {"source_path": "/path/to/exiftool.xml"}
+        # {"source_path": "/path/to/exiftool_xmls/"}
         if option.source_path is not None:
             return index_rdf_description_by_path([option.source_path])
 
-        elif option.pattern is not None:
+        # Find RDF descriptions by pattern matching
+        # i.e. "video.mp4" matches "/path/to/video.xml" regardless of "rdf:about"
+        # {"pattern": "/path/to/%g.xml"}
+        if option.pattern is not None:
             rdf_by_path = {}
             for path in paths:
-                source_path = option.resolve(path)
-                r = index_rdf_description_by_path([source_path])
-                rdfs = list(r.values())
-                if rdfs:
-                    rdf_by_path[exiftool_read.canonical_path(path)] = rdfs[0]
+                canonical_path = exiftool_read.canonical_path(path)
+
+                # Skip non-existent resolved source paths to avoid verbose warnings
+                resolved_source_path = option.resolve(path)
+                if not resolved_source_path.exists():
+                    continue
+
+                rdf_by_about = index_rdf_description_by_path([resolved_source_path])
+                if not rdf_by_about:
+                    continue
+
+                rdf = rdf_by_about.get(canonical_path)
+                if rdf is None:
+                    about, rdf = list(rdf_by_about.items())[0]
+                    if len(rdf_by_about) > 1:
+                        LOG.warning(
+                            f"Found {len(rdf_by_about)} RDFs in the XML source {resolved_source_path}. Using the first RDF (with rdf:about={about}) for {path}"
+                        )
+                rdf_by_path[canonical_path] = rdf
+
             return rdf_by_path
 
-        else:
-            assert False, "Either source_path or pattern must be provided"
+        raise AssertionError("Either source_path or pattern must be provided")
 
 
 class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
@@ -110,7 +131,7 @@ class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
             LOG.warning(
                 "Failed to parse ExifTool XML: %s",
                 str(ex),
-                exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
             )
             rdf_by_path = {}
         else:

mapillary_tools/geotag/options.py

@@ -173,8 +173,11 @@ SourceOptionSchema = {
 }
 
 
+SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
+
+
 def validate_option(instance):
-    jsonschema.validate(instance=instance, schema=SourceOptionSchema)
+    SourceOptionSchemaValidator.validate(instance=instance)
 
 
 if __name__ == "__main__":

mapillary_tools/geotag/utils.py

@@ -37,26 +37,23 @@ def parse_gpx(gpx_file: Path) -> list[Track]:
     return tracks
 
 
-def index_rdf_description_by_path(
-    xml_paths: T.Sequence[Path],
-) -> dict[str, ET.Element]:
-    rdf_description_by_path: dict[str, ET.Element] = {}
+def index_rdf_description_by_path(xml_paths: T.Sequence[Path]) -> dict[str, ET.Element]:
+    rdf_by_path: dict[str, ET.Element] = {}
 
     for xml_path in utils.find_xml_files(xml_paths):
         try:
             etree = ET.parse(xml_path)
-        except ET.ParseError as ex:
-            verbose = LOG.getEffectiveLevel() <= logging.DEBUG
-            if verbose:
-                LOG.warning("Failed to parse %s", xml_path, exc_info=True)
-            else:
-                LOG.warning("Failed to parse %s: %s", xml_path, ex)
+        except Exception as ex:
+            LOG.warning(
+                f"Failed to parse {xml_path}: {ex}",
+                exc_info=LOG.isEnabledFor(logging.DEBUG),
+            )
             continue
 
-        rdf_description_by_path.update(
+        rdf_by_path.update(
             exiftool_read.index_rdf_description_by_path_from_xml_element(
                 etree.getroot()
             )
         )
 
-    return rdf_description_by_path
+    return rdf_by_path

mapillary_tools/geotag/video_extractors/gpx.py

@@ -12,7 +12,7 @@ if sys.version_info >= (3, 12):
 else:
     from typing_extensions import override
 
-from ... import exceptions, geo, telemetry, types
+from ... import exceptions, geo, telemetry, types, utils
 from ..utils import parse_gpx
 from .base import BaseVideoExtractor
 from .native import NativeVideoExtractor
@@ -59,6 +59,7 @@ class GPXVideoExtractor(BaseVideoExtractor):
             self._rebase_times(gpx_points)
             return types.VideoMetadata(
                 filename=self.video_path,
+                filesize=utils.get_file_size(self.video_path),
                 filetype=types.FileType.VIDEO,
                 points=gpx_points,
             )

mapillary_tools/geotag/video_extractors/native.py

@@ -12,6 +12,7 @@ else:
 from ... import blackvue_parser, exceptions, geo, telemetry, types, utils
 from ...camm import camm_parser
 from ...gpmf import gpmf_gps_filter, gpmf_parser
+from ...mp4 import construct_mp4_parser, simple_mp4_parser
 from .base import BaseVideoExtractor
 
 
@@ -113,6 +114,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
             extractor = GoProVideoExtractor(self.video_path)
             try:
                 return extractor.extract()
+            except simple_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
+            except construct_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
             except exceptions.MapillaryVideoGPSNotFoundError:
                 pass
 
@@ -120,6 +129,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
             extractor = CAMMVideoExtractor(self.video_path)
             try:
                 return extractor.extract()
+            except simple_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
+            except construct_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
             except exceptions.MapillaryVideoGPSNotFoundError:
                 pass
 
@@ -127,6 +144,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
             extractor = BlackVueVideoExtractor(self.video_path)
             try:
                 return extractor.extract()
+            except simple_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
+            except construct_mp4_parser.BoxNotFoundError as ex:
+                raise exceptions.MapillaryInvalidVideoError(
+                    f"Invalid video: {ex}"
+                ) from ex
             except exceptions.MapillaryVideoGPSNotFoundError:
                 pass