mapillary-tools 0.14.0b1__py3-none-any.whl → 0.14.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +66 -263
  3. mapillary_tools/authenticate.py +46 -38
  4. mapillary_tools/commands/__main__.py +15 -16
  5. mapillary_tools/commands/upload.py +33 -4
  6. mapillary_tools/constants.py +127 -45
  7. mapillary_tools/exceptions.py +4 -0
  8. mapillary_tools/exif_read.py +2 -1
  9. mapillary_tools/exif_write.py +3 -1
  10. mapillary_tools/geo.py +16 -0
  11. mapillary_tools/geotag/base.py +6 -2
  12. mapillary_tools/geotag/factory.py +9 -1
  13. mapillary_tools/geotag/geotag_images_from_exiftool.py +1 -1
  14. mapillary_tools/geotag/geotag_images_from_gpx.py +0 -6
  15. mapillary_tools/geotag/geotag_videos_from_exiftool.py +30 -9
  16. mapillary_tools/geotag/utils.py +9 -12
  17. mapillary_tools/geotag/video_extractors/gpx.py +2 -1
  18. mapillary_tools/geotag/video_extractors/native.py +25 -0
  19. mapillary_tools/history.py +124 -7
  20. mapillary_tools/http.py +211 -0
  21. mapillary_tools/mp4/construct_mp4_parser.py +8 -2
  22. mapillary_tools/process_geotag_properties.py +31 -27
  23. mapillary_tools/process_sequence_properties.py +339 -322
  24. mapillary_tools/sample_video.py +1 -2
  25. mapillary_tools/serializer/description.py +56 -56
  26. mapillary_tools/serializer/gpx.py +1 -1
  27. mapillary_tools/upload.py +201 -205
  28. mapillary_tools/upload_api_v4.py +57 -47
  29. mapillary_tools/uploader.py +720 -285
  30. mapillary_tools/utils.py +57 -5
  31. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/METADATA +7 -6
  32. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/RECORD +36 -35
  33. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/WHEEL +0 -0
  34. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/entry_points.txt +0 -0
  35. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/licenses/LICENSE +0 -0
  36. {mapillary_tools-0.14.0b1.dist-info → mapillary_tools-0.14.1.dist-info}/top_level.txt +0 -0
@@ -1,6 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import functools
3
4
  import os
5
+ import tempfile
4
6
 
5
7
  import appdirs
6
8
 
@@ -8,44 +10,92 @@ _ENV_PREFIX = "MAPILLARY_TOOLS_"
8
10
 
9
11
 
10
12
  def _yes_or_no(val: str) -> bool:
11
- return val.strip().upper() in [
12
- "1",
13
- "TRUE",
14
- "YES",
15
- ]
13
+ return val.strip().upper() in ["1", "TRUE", "YES"]
16
14
 
17
15
 
18
- # In meters
19
- CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
16
+ def _parse_scaled_integers(
17
+ value: str, scale: dict[str, int] | None = None
18
+ ) -> int | None:
19
+ """
20
+ >>> scale = {"": 1, "b": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024}
21
+ >>> _parse_scaled_integers("0", scale=scale)
22
+ 0
23
+ >>> _parse_scaled_integers("10", scale=scale)
24
+ 10
25
+ >>> _parse_scaled_integers("100B", scale=scale)
26
+ 100
27
+ >>> _parse_scaled_integers("100k", scale=scale)
28
+ 102400
29
+ >>> _parse_scaled_integers("100t", scale=scale)
30
+ Traceback (most recent call last):
31
+ ValueError: Expect valid integer ends with , b, K, M, G, but got 100T
32
+ """
33
+
34
+ if scale is None:
35
+ scale = {"": 1}
36
+
37
+ value = value.strip().upper()
38
+
39
+ if value in ["INF", "INFINITY"]:
40
+ return None
41
+
42
+ try:
43
+ for k, v in scale.items():
44
+ k = k.upper()
45
+ if k and value.endswith(k):
46
+ return int(value[: -len(k)]) * v
47
+
48
+ if "" in scale:
49
+ return int(value) * scale[""]
50
+ except ValueError:
51
+ pass
52
+
53
+ raise ValueError(
54
+ f"Expect valid integer ends with {', '.join(scale.keys())}, but got {value}"
55
+ )
56
+
57
+
58
+ _parse_pixels = functools.partial(
59
+ _parse_scaled_integers,
60
+ scale={
61
+ "": 1,
62
+ "K": 1000,
63
+ "M": 1000 * 1000,
64
+ "MP": 1000 * 1000,
65
+ "G": 1000 * 1000 * 1000,
66
+ "GP": 1000 * 1000 * 1000,
67
+ },
68
+ )
69
+
70
+ _parse_filesize = functools.partial(
71
+ _parse_scaled_integers,
72
+ scale={"B": 1, "K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024},
73
+ )
74
+
75
+ ###################
76
+ ##### GENERAL #####
77
+ ###################
78
+ USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
79
+ PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
80
+
81
+
82
+ ############################
83
+ ##### VIDEO PROCESSING #####
84
+ ############################
20
85
  # In seconds
21
- CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
22
- DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
23
- DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
24
- MAX_AVG_SPEED = float(
25
- os.getenv(_ENV_PREFIX + "MAX_AVG_SPEED", 400_000 / 3600)
26
- ) # 400 KM/h
27
- # in seconds
28
86
  VIDEO_SAMPLE_INTERVAL = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_INTERVAL", -1))
29
- # in meters
87
+ # In meters
30
88
  VIDEO_SAMPLE_DISTANCE = float(os.getenv(_ENV_PREFIX + "VIDEO_SAMPLE_DISTANCE", 3))
31
89
  VIDEO_DURATION_RATIO = float(os.getenv(_ENV_PREFIX + "VIDEO_DURATION_RATIO", 1))
32
90
  FFPROBE_PATH: str = os.getenv(_ENV_PREFIX + "FFPROBE_PATH", "ffprobe")
33
91
  FFMPEG_PATH: str = os.getenv(_ENV_PREFIX + "FFMPEG_PATH", "ffmpeg")
34
- # When not set, MT will try to check both "exiftool" and "exiftool.exe" from $PATH
35
- EXIFTOOL_PATH: str | None = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH")
92
+ EXIFTOOL_PATH: str = os.getenv(_ENV_PREFIX + "EXIFTOOL_PATH", "exiftool")
36
93
  IMAGE_DESCRIPTION_FILENAME = os.getenv(
37
94
  _ENV_PREFIX + "IMAGE_DESCRIPTION_FILENAME", "mapillary_image_description.json"
38
95
  )
39
96
  SAMPLED_VIDEO_FRAMES_FILENAME = os.getenv(
40
97
  _ENV_PREFIX + "SAMPLED_VIDEO_FRAMES_FILENAME", "mapillary_sampled_video_frames"
41
98
  )
42
- USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary")
43
- # The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
44
- # for uploading data to MLY upload service.
45
- # Changing this size does not change the number of requests nor affect upload performance,
46
- # but it affects the responsiveness of the upload progress bar
47
- UPLOAD_CHUNK_SIZE_MB = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
48
-
49
99
  # DoP value, the lower the better
50
100
  # See https://github.com/gopro/gpmf-parser#hero5-black-with-gps-enabled-adds
51
101
  # It is used to filter out noisy points
@@ -54,40 +104,72 @@ GOPRO_MAX_DOP100 = int(os.getenv(_ENV_PREFIX + "GOPRO_MAX_DOP100", 1000))
54
104
  GOPRO_GPS_FIXES: set[int] = set(
55
105
  int(fix) for fix in os.getenv(_ENV_PREFIX + "GOPRO_GPS_FIXES", "2,3").split(",")
56
106
  )
57
- MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
58
-
59
107
  # GPS precision, in meters, is used to filter outliers
60
108
  GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
109
+ MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
110
+ os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
111
+ )
61
112
 
113
+
114
+ #################################
115
+ ###### SEQUENCE PROCESSING ######
116
+ #################################
117
+ # In meters
118
+ CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600))
119
+ # In seconds
120
+ CUTOFF_TIME = float(os.getenv(_ENV_PREFIX + "CUTOFF_TIME", 60))
121
+ DUPLICATE_DISTANCE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_DISTANCE", 0.1))
122
+ DUPLICATE_ANGLE = float(os.getenv(_ENV_PREFIX + "DUPLICATE_ANGLE", 5))
123
+ MAX_CAPTURE_SPEED_KMH = float(
124
+ os.getenv(_ENV_PREFIX + "MAX_CAPTURE_SPEED_KMH", 400)
125
+ ) # 400 KM/h
62
126
  # WARNING: Changing the following envvars might result in failed uploads
63
127
  # Max number of images per sequence
64
- MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
128
+ MAX_SEQUENCE_LENGTH: int | None = _parse_scaled_integers(
129
+ os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", "1000")
130
+ )
65
131
  # Max file size per sequence (sum of image filesizes in the sequence)
66
- MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
132
+ MAX_SEQUENCE_FILESIZE: int | None = _parse_filesize(
133
+ os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G")
134
+ )
67
135
  # Max number of pixels per sequence (sum of image pixels in the sequence)
68
- MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
69
-
70
- PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO"))
71
-
72
- _AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
73
- os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
136
+ MAX_SEQUENCE_PIXELS: int | None = _parse_pixels(
137
+ os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
74
138
  )
75
139
 
140
+
141
+ ##################
142
+ ##### UPLOAD #####
143
+ ##################
76
144
  MAPILLARY_DISABLE_API_LOGGING: bool = _yes_or_no(
77
145
  os.getenv("MAPILLARY_DISABLE_API_LOGGING", "NO")
78
146
  )
147
+ MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
148
+ "MAPILLARY_UPLOAD_HISTORY_PATH", os.path.join(USER_DATA_DIR, "upload_history")
149
+ )
150
+ UPLOAD_CACHE_DIR: str = os.getenv(
151
+ _ENV_PREFIX + "UPLOAD_CACHE_DIR",
152
+ os.path.join(tempfile.gettempdir(), "mapillary_tools", "upload_cache"),
153
+ )
154
+ # The minimal upload speed is used to calculate the read timeout to avoid upload hanging:
155
+ # timeout = upload_size / MIN_UPLOAD_SPEED
156
+ MIN_UPLOAD_SPEED: int | None = _parse_filesize(
157
+ os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K") # 50 Kb/s
158
+ )
159
+ # Maximum number of parallel workers for uploading images within a single sequence.
160
+ # NOTE: Sequences themselves are uploaded sequentially, not in parallel.
161
+ MAX_IMAGE_UPLOAD_WORKERS: int = int(
162
+ os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 4)
163
+ )
164
+ # The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
165
+ # for uploading data to MLY upload service.
166
+ # Changing this size does not change the number of requests nor affect upload performance,
167
+ # but it affects the responsiveness of the upload progress bar
168
+ UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 2))
169
+ MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
79
170
  MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
80
171
  os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
81
172
  )
82
- MAPILLARY__EXPERIMENTAL_ENABLE_IMU: bool = _yes_or_no(
83
- os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU", "NO")
84
- )
85
- MAPILLARY_UPLOAD_HISTORY_PATH: str = os.getenv(
86
- "MAPILLARY_UPLOAD_HISTORY_PATH",
87
- os.path.join(
88
- USER_DATA_DIR,
89
- "upload_history",
90
- ),
173
+ _AUTH_VERIFICATION_DISABLED: bool = _yes_or_no(
174
+ os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO")
91
175
  )
92
-
93
- MAX_IMAGE_UPLOAD_WORKERS = int(os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 64))
@@ -51,6 +51,10 @@ class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError):
51
51
  pass
52
52
 
53
53
 
54
+ class MapillaryInvalidVideoError(MapillaryDescriptionError):
55
+ pass
56
+
57
+
54
58
  class MapillaryGPXEmptyError(MapillaryDescriptionError):
55
59
  pass
56
60
 
@@ -871,7 +871,8 @@ class ExifRead(ExifReadFromEXIF):
871
871
 
872
872
  def _xmp_with_reason(self, reason: str) -> ExifReadFromXMP | None:
873
873
  if not self._xml_extracted:
874
- LOG.debug('Extracting XMP for "%s"', reason)
874
+ # TODO Disabled because too verbose but still useful to know
875
+ # LOG.debug('Extracting XMP for "%s"', reason)
875
876
  self._cached_xml = self._extract_xmp()
876
877
  self._xml_extracted = True
877
878
 
@@ -42,7 +42,9 @@ class ExifEdit:
42
42
 
43
43
  def add_image_description(self, data: dict) -> None:
44
44
  """Add a dict to image description."""
45
- self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(data)
45
+ self._ef["0th"][piexif.ImageIFD.ImageDescription] = json.dumps(
46
+ data, sort_keys=True, separators=(",", ":")
47
+ )
46
48
 
47
49
  def add_orientation(self, orientation: int) -> None:
48
50
  """Add image orientation to image."""
mapillary_tools/geo.py CHANGED
@@ -51,6 +51,22 @@ def gps_distance(latlon_1: tuple[float, float], latlon_2: tuple[float, float]) -
51
51
  return math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2 + (z1 - z2) ** 2)
52
52
 
53
53
 
54
+ def avg_speed(sequence: T.Sequence[PointLike]) -> float:
55
+ total_distance = 0.0
56
+ for cur, nxt in pairwise(sequence):
57
+ total_distance += gps_distance((cur.lat, cur.lon), (nxt.lat, nxt.lon))
58
+
59
+ if sequence:
60
+ time_diff = sequence[-1].time - sequence[0].time
61
+ else:
62
+ time_diff = 0.0
63
+
64
+ if time_diff == 0.0:
65
+ return float("inf")
66
+
67
+ return total_distance / time_diff
68
+
69
+
54
70
  def compute_bearing(
55
71
  latlon_1: tuple[float, float],
56
72
  latlon_2: tuple[float, float],
@@ -46,7 +46,7 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
46
46
  map_results,
47
47
  desc="Extracting images",
48
48
  unit="images",
49
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
49
+ disable=LOG.isEnabledFor(logging.DEBUG),
50
50
  total=len(extractors),
51
51
  )
52
52
  )
@@ -62,6 +62,8 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
62
62
  try:
63
63
  return extractor.extract()
64
64
  except exceptions.MapillaryDescriptionError as ex:
65
+ if LOG.isEnabledFor(logging.DEBUG):
66
+ LOG.error(f"{cls.__name__}({image_path.name}): {ex}")
65
67
  return types.describe_error_metadata(
66
68
  ex, image_path, filetype=types.FileType.IMAGE
67
69
  )
@@ -112,7 +114,7 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
112
114
  map_results,
113
115
  desc="Extracting videos",
114
116
  unit="videos",
115
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
117
+ disable=LOG.isEnabledFor(logging.DEBUG),
116
118
  total=len(extractors),
117
119
  )
118
120
  )
@@ -128,6 +130,8 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
128
130
  try:
129
131
  return extractor.extract()
130
132
  except exceptions.MapillaryDescriptionError as ex:
133
+ if LOG.isEnabledFor(logging.DEBUG):
134
+ LOG.error(f"{cls.__name__}({video_path.name}): {ex}")
131
135
  return types.describe_error_metadata(
132
136
  ex, video_path, filetype=types.FileType.VIDEO
133
137
  )
@@ -67,7 +67,14 @@ def process(
67
67
  reprocessable_paths = set(paths)
68
68
 
69
69
  for idx, option in enumerate(options):
70
- LOG.debug("Processing %d files with %s", len(reprocessable_paths), option)
70
+ if LOG.isEnabledFor(logging.DEBUG):
71
+ LOG.info(
72
+ f"==> Processing {len(reprocessable_paths)} files with source {option}..."
73
+ )
74
+ else:
75
+ LOG.info(
76
+ f"==> Processing {len(reprocessable_paths)} files with source {option.source.value}..."
77
+ )
71
78
 
72
79
  image_videos, video_paths = _filter_images_and_videos(
73
80
  reprocessable_paths, option.filetypes
@@ -114,6 +121,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
114
121
  exceptions.MapillaryGeoTaggingError,
115
122
  exceptions.MapillaryVideoGPSNotFoundError,
116
123
  exceptions.MapillaryExiftoolNotFoundError,
124
+ exceptions.MapillaryExifToolXMLNotFoundError,
117
125
  ),
118
126
  ):
119
127
  return True
@@ -93,7 +93,7 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
93
93
  LOG.warning(
94
94
  "Failed to parse ExifTool XML: %s",
95
95
  str(ex),
96
- exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
96
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
97
97
  )
98
98
  rdf_by_path = {}
99
99
  else:
@@ -20,12 +20,6 @@ from .geotag_images_from_exif import ImageEXIFExtractor
20
20
  LOG = logging.getLogger(__name__)
21
21
 
22
22
 
23
- class SyncMode:
24
- SYNC = "sync"
25
- STRICT_SYNC = "strict_sync"
26
- RESET = "reset"
27
-
28
-
29
23
  class GeotagImagesFromGPX(GeotagImagesFromGeneric):
30
24
  def __init__(
31
25
  self,
@@ -61,21 +61,42 @@ class GeotagVideosFromExifToolXML(GeotagVideosFromGeneric):
61
61
  def find_rdf_by_path(
62
62
  cls, option: options.SourcePathOption, paths: T.Iterable[Path]
63
63
  ) -> dict[str, ET.Element]:
64
+ # Find RDF descriptions by path in RDF description
65
+ # Sources are matched based on the paths in "rdf:about" in XML elements
66
+ # {"source_path": "/path/to/exiftool.xml"}
67
+ # {"source_path": "/path/to/exiftool_xmls/"}
64
68
  if option.source_path is not None:
65
69
  return index_rdf_description_by_path([option.source_path])
66
70
 
67
- elif option.pattern is not None:
71
+ # Find RDF descriptions by pattern matching
72
+ # i.e. "video.mp4" matches "/path/to/video.xml" regardless of "rdf:about"
73
+ # {"pattern": "/path/to/%g.xml"}
74
+ if option.pattern is not None:
68
75
  rdf_by_path = {}
69
76
  for path in paths:
70
- source_path = option.resolve(path)
71
- r = index_rdf_description_by_path([source_path])
72
- rdfs = list(r.values())
73
- if rdfs:
74
- rdf_by_path[exiftool_read.canonical_path(path)] = rdfs[0]
77
+ canonical_path = exiftool_read.canonical_path(path)
78
+
79
+ # Skip non-existent resolved source paths to avoid verbose warnings
80
+ resolved_source_path = option.resolve(path)
81
+ if not resolved_source_path.exists():
82
+ continue
83
+
84
+ rdf_by_about = index_rdf_description_by_path([resolved_source_path])
85
+ if not rdf_by_about:
86
+ continue
87
+
88
+ rdf = rdf_by_about.get(canonical_path)
89
+ if rdf is None:
90
+ about, rdf = list(rdf_by_about.items())[0]
91
+ if len(rdf_by_about) > 1:
92
+ LOG.warning(
93
+ f"Found {len(rdf_by_about)} RDFs in the XML source {resolved_source_path}. Using the first RDF (with rdf:about={about}) for {path}"
94
+ )
95
+ rdf_by_path[canonical_path] = rdf
96
+
75
97
  return rdf_by_path
76
98
 
77
- else:
78
- assert False, "Either source_path or pattern must be provided"
99
+ raise AssertionError("Either source_path or pattern must be provided")
79
100
 
80
101
 
81
102
  class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
@@ -110,7 +131,7 @@ class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
110
131
  LOG.warning(
111
132
  "Failed to parse ExifTool XML: %s",
112
133
  str(ex),
113
- exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
134
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
114
135
  )
115
136
  rdf_by_path = {}
116
137
  else:
@@ -37,26 +37,23 @@ def parse_gpx(gpx_file: Path) -> list[Track]:
37
37
  return tracks
38
38
 
39
39
 
40
- def index_rdf_description_by_path(
41
- xml_paths: T.Sequence[Path],
42
- ) -> dict[str, ET.Element]:
43
- rdf_description_by_path: dict[str, ET.Element] = {}
40
+ def index_rdf_description_by_path(xml_paths: T.Sequence[Path]) -> dict[str, ET.Element]:
41
+ rdf_by_path: dict[str, ET.Element] = {}
44
42
 
45
43
  for xml_path in utils.find_xml_files(xml_paths):
46
44
  try:
47
45
  etree = ET.parse(xml_path)
48
- except ET.ParseError as ex:
49
- verbose = LOG.getEffectiveLevel() <= logging.DEBUG
50
- if verbose:
51
- LOG.warning("Failed to parse %s", xml_path, exc_info=True)
52
- else:
53
- LOG.warning("Failed to parse %s: %s", xml_path, ex)
46
+ except Exception as ex:
47
+ LOG.warning(
48
+ f"Failed to parse {xml_path}: {ex}",
49
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
50
+ )
54
51
  continue
55
52
 
56
- rdf_description_by_path.update(
53
+ rdf_by_path.update(
57
54
  exiftool_read.index_rdf_description_by_path_from_xml_element(
58
55
  etree.getroot()
59
56
  )
60
57
  )
61
58
 
62
- return rdf_description_by_path
59
+ return rdf_by_path
@@ -12,7 +12,7 @@ if sys.version_info >= (3, 12):
12
12
  else:
13
13
  from typing_extensions import override
14
14
 
15
- from ... import exceptions, geo, telemetry, types
15
+ from ... import exceptions, geo, telemetry, types, utils
16
16
  from ..utils import parse_gpx
17
17
  from .base import BaseVideoExtractor
18
18
  from .native import NativeVideoExtractor
@@ -59,6 +59,7 @@ class GPXVideoExtractor(BaseVideoExtractor):
59
59
  self._rebase_times(gpx_points)
60
60
  return types.VideoMetadata(
61
61
  filename=self.video_path,
62
+ filesize=utils.get_file_size(self.video_path),
62
63
  filetype=types.FileType.VIDEO,
63
64
  points=gpx_points,
64
65
  )
@@ -12,6 +12,7 @@ else:
12
12
  from ... import blackvue_parser, exceptions, geo, telemetry, types, utils
13
13
  from ...camm import camm_parser
14
14
  from ...gpmf import gpmf_gps_filter, gpmf_parser
15
+ from ...mp4 import construct_mp4_parser, simple_mp4_parser
15
16
  from .base import BaseVideoExtractor
16
17
 
17
18
 
@@ -113,6 +114,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
113
114
  extractor = GoProVideoExtractor(self.video_path)
114
115
  try:
115
116
  return extractor.extract()
117
+ except simple_mp4_parser.BoxNotFoundError as ex:
118
+ raise exceptions.MapillaryInvalidVideoError(
119
+ f"Invalid video: {ex}"
120
+ ) from ex
121
+ except construct_mp4_parser.BoxNotFoundError as ex:
122
+ raise exceptions.MapillaryInvalidVideoError(
123
+ f"Invalid video: {ex}"
124
+ ) from ex
116
125
  except exceptions.MapillaryVideoGPSNotFoundError:
117
126
  pass
118
127
 
@@ -120,6 +129,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
120
129
  extractor = CAMMVideoExtractor(self.video_path)
121
130
  try:
122
131
  return extractor.extract()
132
+ except simple_mp4_parser.BoxNotFoundError as ex:
133
+ raise exceptions.MapillaryInvalidVideoError(
134
+ f"Invalid video: {ex}"
135
+ ) from ex
136
+ except construct_mp4_parser.BoxNotFoundError as ex:
137
+ raise exceptions.MapillaryInvalidVideoError(
138
+ f"Invalid video: {ex}"
139
+ ) from ex
123
140
  except exceptions.MapillaryVideoGPSNotFoundError:
124
141
  pass
125
142
 
@@ -127,6 +144,14 @@ class NativeVideoExtractor(BaseVideoExtractor):
127
144
  extractor = BlackVueVideoExtractor(self.video_path)
128
145
  try:
129
146
  return extractor.extract()
147
+ except simple_mp4_parser.BoxNotFoundError as ex:
148
+ raise exceptions.MapillaryInvalidVideoError(
149
+ f"Invalid video: {ex}"
150
+ ) from ex
151
+ except construct_mp4_parser.BoxNotFoundError as ex:
152
+ raise exceptions.MapillaryInvalidVideoError(
153
+ f"Invalid video: {ex}"
154
+ ) from ex
130
155
  except exceptions.MapillaryVideoGPSNotFoundError:
131
156
  pass
132
157