mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +5 -4
  3. mapillary_tools/authenticate.py +9 -9
  4. mapillary_tools/blackvue_parser.py +79 -22
  5. mapillary_tools/camm/camm_parser.py +5 -5
  6. mapillary_tools/commands/__main__.py +1 -2
  7. mapillary_tools/config.py +41 -18
  8. mapillary_tools/constants.py +3 -2
  9. mapillary_tools/exceptions.py +1 -1
  10. mapillary_tools/exif_read.py +65 -65
  11. mapillary_tools/exif_write.py +7 -7
  12. mapillary_tools/exiftool_read.py +23 -46
  13. mapillary_tools/exiftool_read_video.py +88 -49
  14. mapillary_tools/exiftool_runner.py +4 -24
  15. mapillary_tools/ffmpeg.py +417 -242
  16. mapillary_tools/geo.py +4 -21
  17. mapillary_tools/geotag/__init__.py +0 -1
  18. mapillary_tools/geotag/{geotag_from_generic.py → base.py} +34 -50
  19. mapillary_tools/geotag/factory.py +105 -103
  20. mapillary_tools/geotag/geotag_images_from_exif.py +15 -51
  21. mapillary_tools/geotag/geotag_images_from_exiftool.py +118 -63
  22. mapillary_tools/geotag/geotag_images_from_gpx.py +33 -16
  23. mapillary_tools/geotag/geotag_images_from_gpx_file.py +2 -34
  24. mapillary_tools/geotag/geotag_images_from_nmea_file.py +0 -3
  25. mapillary_tools/geotag/geotag_images_from_video.py +51 -14
  26. mapillary_tools/geotag/geotag_videos_from_exiftool.py +123 -0
  27. mapillary_tools/geotag/geotag_videos_from_gpx.py +35 -123
  28. mapillary_tools/geotag/geotag_videos_from_video.py +14 -147
  29. mapillary_tools/geotag/image_extractors/base.py +18 -0
  30. mapillary_tools/geotag/image_extractors/exif.py +60 -0
  31. mapillary_tools/geotag/image_extractors/exiftool.py +18 -0
  32. mapillary_tools/geotag/options.py +26 -3
  33. mapillary_tools/geotag/utils.py +62 -0
  34. mapillary_tools/geotag/video_extractors/base.py +18 -0
  35. mapillary_tools/geotag/video_extractors/exiftool.py +70 -0
  36. mapillary_tools/geotag/video_extractors/gpx.py +116 -0
  37. mapillary_tools/geotag/video_extractors/native.py +135 -0
  38. mapillary_tools/gpmf/gpmf_parser.py +16 -16
  39. mapillary_tools/gpmf/gps_filter.py +5 -3
  40. mapillary_tools/history.py +8 -3
  41. mapillary_tools/mp4/construct_mp4_parser.py +9 -8
  42. mapillary_tools/mp4/mp4_sample_parser.py +27 -27
  43. mapillary_tools/mp4/simple_mp4_builder.py +10 -9
  44. mapillary_tools/mp4/simple_mp4_parser.py +13 -12
  45. mapillary_tools/process_geotag_properties.py +21 -15
  46. mapillary_tools/process_sequence_properties.py +49 -49
  47. mapillary_tools/sample_video.py +15 -14
  48. mapillary_tools/serializer/description.py +587 -0
  49. mapillary_tools/serializer/gpx.py +132 -0
  50. mapillary_tools/telemetry.py +6 -5
  51. mapillary_tools/types.py +64 -635
  52. mapillary_tools/upload.py +176 -197
  53. mapillary_tools/upload_api_v4.py +94 -51
  54. mapillary_tools/uploader.py +284 -138
  55. mapillary_tools/utils.py +16 -18
  56. {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0b1.dist-info}/METADATA +87 -31
  57. mapillary_tools-0.14.0b1.dist-info/RECORD +75 -0
  58. {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0b1.dist-info}/WHEEL +1 -1
  59. mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -77
  60. mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +0 -151
  61. mapillary_tools/video_data_extraction/cli_options.py +0 -22
  62. mapillary_tools/video_data_extraction/extract_video_data.py +0 -157
  63. mapillary_tools/video_data_extraction/extractors/base_parser.py +0 -75
  64. mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +0 -49
  65. mapillary_tools/video_data_extraction/extractors/camm_parser.py +0 -62
  66. mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +0 -74
  67. mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +0 -52
  68. mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +0 -52
  69. mapillary_tools/video_data_extraction/extractors/gopro_parser.py +0 -58
  70. mapillary_tools/video_data_extraction/extractors/gpx_parser.py +0 -108
  71. mapillary_tools/video_data_extraction/extractors/nmea_parser.py +0 -24
  72. mapillary_tools/video_data_extraction/video_data_parser_factory.py +0 -39
  73. mapillary_tools-0.14.0a1.dist-info/RECORD +0 -78
  74. {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0b1.dist-info}/entry_points.txt +0 -0
  75. {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0b1.dist-info}/licenses/LICENSE +0 -0
  76. {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0b1.dist-info}/top_level.txt +0 -0
mapillary_tools/types.py CHANGED
@@ -1,35 +1,14 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import abc
3
4
  import dataclasses
4
- import datetime
5
5
  import enum
6
6
  import hashlib
7
- import json
8
- import os
9
7
  import typing as T
10
8
  import uuid
11
9
  from pathlib import Path
12
- from typing import Literal, TypedDict
13
10
 
14
- import jsonschema
15
-
16
- from . import exceptions, geo, utils
17
-
18
-
19
- # http://wiki.gis.com/wiki/index.php/Decimal_degrees
20
- # decimal places degrees distance
21
- # 0 1.0 111 km
22
- # 1 0.1 11.1 km
23
- # 2 0.01 1.11 km
24
- # 3 0.001 111 m
25
- # 4 0.0001 11.1 m
26
- # 5 0.00001 1.11 m
27
- # 6 0.000001 0.111 m
28
- # 7 0.0000001 1.11 cm
29
- # 8 0.00000001 1.11 mm
30
- _COORDINATES_PRECISION = 7
31
- _ALTITUDE_PRECISION = 3
32
- _ANGLE_PRECISION = 3
11
+ from . import geo, utils
33
12
 
34
13
 
35
14
  class FileType(enum.Enum):
@@ -53,23 +32,23 @@ NATIVE_VIDEO_FILETYPES = {
53
32
  @dataclasses.dataclass
54
33
  class ImageMetadata(geo.Point):
55
34
  filename: Path
56
- # if None or absent, it will be calculated
57
- md5sum: T.Optional[str] = None
58
- # filetype: is always FileType.IMAGE
59
- width: T.Optional[int] = None
60
- height: T.Optional[int] = None
61
- MAPSequenceUUID: T.Optional[str] = None
62
- MAPDeviceMake: T.Optional[str] = None
63
- MAPDeviceModel: T.Optional[str] = None
64
- MAPGPSAccuracyMeters: T.Optional[float] = None
65
- MAPCameraUUID: T.Optional[str] = None
66
- MAPOrientation: T.Optional[int] = None
67
- # deprecated since v0.10.0; keep here for compatibility
68
- MAPMetaTags: T.Optional[T.Dict] = None
69
- MAPFilename: T.Optional[str] = None
70
- filesize: T.Optional[int] = None
71
-
72
- def update_md5sum(self, image_data: T.Optional[T.BinaryIO] = None) -> None:
35
+ # filetype should be always FileType.IMAGE
36
+ md5sum: str | None = None
37
+ width: int | None = None
38
+ height: int | None = None
39
+ filesize: int | None = None
40
+
41
+ # Fields starting with MAP* will be written to the image EXIF
42
+ MAPSequenceUUID: str | None = None
43
+ MAPDeviceMake: str | None = None
44
+ MAPDeviceModel: str | None = None
45
+ MAPGPSAccuracyMeters: float | None = None
46
+ MAPCameraUUID: str | None = None
47
+ MAPOrientation: int | None = None
48
+ MAPMetaTags: dict | None = None
49
+ MAPFilename: str | None = None
50
+
51
+ def update_md5sum(self, image_data: T.BinaryIO | None = None) -> None:
73
52
  if self.md5sum is None:
74
53
  if image_data is None:
75
54
  with self.filename.open("rb") as fp:
@@ -87,13 +66,12 @@ class ImageMetadata(geo.Point):
87
66
  @dataclasses.dataclass
88
67
  class VideoMetadata:
89
68
  filename: Path
90
- # if None or absent, it will be calculated
91
69
  filetype: FileType
92
70
  points: T.Sequence[geo.Point]
93
- md5sum: T.Optional[str] = None
94
- make: T.Optional[str] = None
95
- model: T.Optional[str] = None
96
- filesize: T.Optional[int] = None
71
+ md5sum: str | None = None
72
+ make: str | None = None
73
+ model: str | None = None
74
+ filesize: int | None = None
97
75
 
98
76
  def update_md5sum(self) -> None:
99
77
  if self.md5sum is None:
@@ -114,17 +92,47 @@ Metadata = T.Union[ImageMetadata, VideoMetadata]
114
92
  MetadataOrError = T.Union[Metadata, ErrorMetadata]
115
93
 
116
94
 
117
- # Assume {GOPRO, VIDEO} are the NATIVE_VIDEO_FILETYPES:
118
- # a | b = result
119
- # {CAMM} | {GOPRO} = {}
120
- # {CAMM} | {GOPRO, VIDEO} = {CAMM}
121
- # {GOPRO} | {GOPRO, VIDEO} = {GOPRO}
122
- # {GOPRO} | {VIDEO} = {GOPRO}
123
- # {CAMM, GOPRO} | {VIDEO} = {CAMM, GOPRO}
124
- # {VIDEO} | {VIDEO} = {CAMM, GOPRO, VIDEO}
95
+ class BaseSerializer(abc.ABC):
96
+ @classmethod
97
+ @abc.abstractmethod
98
+ def serialize(cls, metadatas: T.Sequence[MetadataOrError]) -> bytes:
99
+ raise NotImplementedError()
100
+
101
+ @classmethod
102
+ @abc.abstractmethod
103
+ def deserialize(cls, data: bytes) -> list[Metadata]:
104
+ raise NotImplementedError()
105
+
106
+ @classmethod
107
+ def deserialize_stream(cls, data: T.IO[bytes]) -> list[Metadata]:
108
+ return cls.deserialize(data.read())
109
+
110
+
125
111
  def combine_filetype_filters(
126
112
  a: set[FileType] | None, b: set[FileType] | None
127
113
  ) -> set[FileType] | None:
114
+ """
115
+ >>> combine_filetype_filters({FileType.CAMM}, {FileType.GOPRO})
116
+ set()
117
+
118
+ >>> combine_filetype_filters({FileType.CAMM}, {FileType.GOPRO, FileType.VIDEO})
119
+ {<FileType.CAMM: 'camm'>}
120
+
121
+ >>> combine_filetype_filters({FileType.GOPRO}, {FileType.GOPRO, FileType.VIDEO})
122
+ {<FileType.GOPRO: 'gopro'>}
123
+
124
+ >>> combine_filetype_filters({FileType.GOPRO}, {FileType.VIDEO})
125
+ {<FileType.GOPRO: 'gopro'>}
126
+
127
+ >>> expected = {FileType.CAMM, FileType.GOPRO}
128
+ >>> combine_filetype_filters({FileType.CAMM, FileType.GOPRO}, {FileType.VIDEO}) == expected
129
+ True
130
+
131
+ >>> expected = {FileType.CAMM, FileType.GOPRO, FileType.BLACKVUE, FileType.VIDEO}
132
+ >>> combine_filetype_filters({FileType.VIDEO}, {FileType.VIDEO}) == expected
133
+ True
134
+ """
135
+
128
136
  if a is None:
129
137
  return b
130
138
 
@@ -143,82 +151,6 @@ def combine_filetype_filters(
143
151
  return a.intersection(b)
144
152
 
145
153
 
146
- class UserItem(TypedDict, total=False):
147
- MAPOrganizationKey: T.Union[int, str]
148
- # Not in use. Keep here for back-compatibility
149
- MAPSettingsUsername: str
150
- MAPSettingsUserKey: str
151
- user_upload_token: str
152
-
153
-
154
- class _CompassHeading(TypedDict, total=True):
155
- TrueHeading: float
156
- MagneticHeading: float
157
-
158
-
159
- class _ImageRequired(TypedDict, total=True):
160
- MAPLatitude: float
161
- MAPLongitude: float
162
- MAPCaptureTime: str
163
-
164
-
165
- class _Image(_ImageRequired, total=False):
166
- MAPAltitude: float
167
- MAPCompassHeading: _CompassHeading
168
-
169
-
170
- class _SequenceOnly(TypedDict, total=False):
171
- MAPSequenceUUID: str
172
-
173
-
174
- class MetaProperties(TypedDict, total=False):
175
- MAPDeviceMake: str
176
- MAPDeviceModel: str
177
- MAPGPSAccuracyMeters: float
178
- MAPCameraUUID: str
179
- MAPOrientation: int
180
-
181
-
182
- class ImageDescription(_SequenceOnly, _Image, MetaProperties, total=True):
183
- # filename is required
184
- filename: str
185
- # if None or absent, it will be calculated
186
- md5sum: T.Optional[str]
187
- filetype: Literal["image"]
188
- filesize: T.Optional[int]
189
-
190
-
191
- class _VideoDescriptionRequired(TypedDict, total=True):
192
- filename: str
193
- # if None or absent, it will be calculated
194
- md5sum: T.Optional[str]
195
- filetype: str
196
- MAPGPSTrack: T.List[T.Sequence[T.Union[float, int, None]]]
197
-
198
-
199
- class VideoDescription(_VideoDescriptionRequired, total=False):
200
- MAPDeviceMake: str
201
- MAPDeviceModel: str
202
- filesize: T.Optional[int]
203
-
204
-
205
- class _ErrorDescription(TypedDict, total=False):
206
- # type and message are required
207
- type: str
208
- message: str
209
- # vars is optional
210
- vars: T.Dict
211
-
212
-
213
- class _ImageDescriptionErrorRequired(TypedDict, total=True):
214
- filename: str
215
- error: _ErrorDescription
216
-
217
-
218
- class ImageDescriptionError(_ImageDescriptionErrorRequired, total=False):
219
- filetype: str
220
-
221
-
222
154
  M = T.TypeVar("M")
223
155
 
224
156
 
@@ -237,516 +169,17 @@ def separate_errors(
237
169
  return good, bad
238
170
 
239
171
 
240
- def _describe_error_desc(
241
- exc: Exception, filename: Path, filetype: T.Optional[FileType]
242
- ) -> ImageDescriptionError:
243
- err: _ErrorDescription = {
244
- "type": exc.__class__.__name__,
245
- "message": str(exc),
246
- }
247
-
248
- exc_vars = vars(exc)
249
-
250
- if exc_vars:
251
- # handle unserializable exceptions
252
- try:
253
- vars_json = json.dumps(exc_vars)
254
- except Exception:
255
- vars_json = ""
256
- if vars_json:
257
- err["vars"] = json.loads(vars_json)
258
-
259
- desc: ImageDescriptionError = {
260
- "error": err,
261
- "filename": str(filename.resolve()),
262
- }
263
- if filetype is not None:
264
- desc["filetype"] = filetype.value
265
-
266
- return desc
267
-
268
-
269
172
  def describe_error_metadata(
270
173
  exc: Exception, filename: Path, filetype: FileType
271
174
  ) -> ErrorMetadata:
272
175
  return ErrorMetadata(filename=filename, filetype=filetype, error=exc)
273
176
 
274
177
 
275
- Description = T.Union[ImageDescription, VideoDescription]
276
- DescriptionOrError = T.Union[ImageDescription, VideoDescription, ImageDescriptionError]
277
-
278
-
279
- UserItemSchema = {
280
- "type": "object",
281
- "properties": {
282
- "MAPOrganizationKey": {"type": ["integer", "string"]},
283
- # Not in use. Keep here for back-compatibility
284
- "MAPSettingsUsername": {"type": "string"},
285
- "MAPSettingsUserKey": {"type": "string"},
286
- "user_upload_token": {"type": "string"},
287
- },
288
- "required": ["user_upload_token"],
289
- "additionalProperties": True,
290
- }
291
-
292
-
293
- ImageDescriptionEXIFSchema = {
294
- "type": "object",
295
- "properties": {
296
- "MAPLatitude": {
297
- "type": "number",
298
- "description": "Latitude of the image",
299
- "minimum": -90,
300
- "maximum": 90,
301
- },
302
- "MAPLongitude": {
303
- "type": "number",
304
- "description": "Longitude of the image",
305
- "minimum": -180,
306
- "maximum": 180,
307
- },
308
- "MAPAltitude": {
309
- "type": "number",
310
- "description": "Altitude of the image, in meters",
311
- },
312
- "MAPCaptureTime": {
313
- "type": "string",
314
- "description": "Capture time of the image",
315
- "pattern": "[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}_[0-9]{2}_[0-9]{2}_[0-9]+",
316
- },
317
- "MAPCompassHeading": {
318
- "type": "object",
319
- "properties": {
320
- "TrueHeading": {"type": "number"},
321
- "MagneticHeading": {"type": "number"},
322
- },
323
- "required": ["TrueHeading", "MagneticHeading"],
324
- "additionalProperties": False,
325
- "description": "Camera angle of the image, in degrees. If null, the angle will be interpolated",
326
- },
327
- "MAPSequenceUUID": {
328
- "type": "string",
329
- "description": "Arbitrary key for grouping images",
330
- "pattern": "[a-zA-Z0-9_-]+",
331
- },
332
- # deprecated since v0.10.0; keep here for compatibility
333
- "MAPMetaTags": {"type": "object"},
334
- "MAPDeviceMake": {"type": "string"},
335
- "MAPDeviceModel": {"type": "string"},
336
- "MAPGPSAccuracyMeters": {"type": "number"},
337
- "MAPCameraUUID": {"type": "string"},
338
- "MAPFilename": {
339
- "type": "string",
340
- "description": "The base filename of the image",
341
- },
342
- "MAPOrientation": {"type": "integer"},
343
- },
344
- "required": [
345
- "MAPLatitude",
346
- "MAPLongitude",
347
- "MAPCaptureTime",
348
- ],
349
- "additionalProperties": False,
350
- }
351
-
352
- VideoDescriptionSchema = {
353
- "type": "object",
354
- "properties": {
355
- "MAPGPSTrack": {
356
- "type": "array",
357
- "items": {
358
- "type": "array",
359
- "description": "track point",
360
- "prefixItems": [
361
- {
362
- "type": "number",
363
- "description": "Time offset of the track point, in milliseconds, relative to the beginning of the video",
364
- },
365
- {
366
- "type": "number",
367
- "description": "Longitude of the track point",
368
- },
369
- {
370
- "type": "number",
371
- "description": "Latitude of the track point",
372
- },
373
- {
374
- "type": ["number", "null"],
375
- "description": "Altitude of the track point in meters",
376
- },
377
- {
378
- "type": ["number", "null"],
379
- "description": "Camera angle of the track point, in degrees. If null, the angle will be interpolated",
380
- },
381
- ],
382
- },
383
- },
384
- "MAPDeviceMake": {
385
- "type": "string",
386
- "description": "Device make, e.g. GoPro, BlackVue, Insta360",
387
- },
388
- "MAPDeviceModel": {
389
- "type": "string",
390
- "description": "Device model, e.g. HERO10 Black, DR900S-1CH, Insta360 Titan",
391
- },
392
- },
393
- "required": [
394
- "MAPGPSTrack",
395
- ],
396
- "additionalProperties": False,
397
- }
398
-
399
-
400
- def merge_schema(*schemas: T.Dict) -> T.Dict:
401
- for s in schemas:
402
- assert s.get("type") == "object", "must be all object schemas"
403
- properties = {}
404
- all_required = []
405
- additional_properties = True
406
- for s in schemas:
407
- properties.update(s.get("properties", {}))
408
- all_required += s.get("required", [])
409
- if "additionalProperties" in s:
410
- additional_properties = s["additionalProperties"]
411
- return {
412
- "type": "object",
413
- "properties": properties,
414
- "required": sorted(set(all_required)),
415
- "additionalProperties": additional_properties,
416
- }
417
-
418
-
419
- ImageDescriptionFileSchema = merge_schema(
420
- ImageDescriptionEXIFSchema,
421
- {
422
- "type": "object",
423
- "properties": {
424
- "filename": {
425
- "type": "string",
426
- "description": "Absolute path of the image",
427
- },
428
- "md5sum": {
429
- "type": ["string", "null"],
430
- "description": "MD5 checksum of the image content. If not provided, the uploader will compute it",
431
- },
432
- "filesize": {
433
- "type": ["number", "null"],
434
- "description": "File size",
435
- },
436
- "filetype": {
437
- "type": "string",
438
- "enum": [FileType.IMAGE.value],
439
- "description": "The image file type",
440
- },
441
- },
442
- "required": [
443
- "filename",
444
- "filetype",
445
- ],
446
- },
447
- )
448
-
449
- VideoDescriptionFileSchema = merge_schema(
450
- VideoDescriptionSchema,
451
- {
452
- "type": "object",
453
- "properties": {
454
- "filename": {
455
- "type": "string",
456
- "description": "Absolute path of the video",
457
- },
458
- "md5sum": {
459
- "type": ["string", "null"],
460
- "description": "MD5 checksum of the video content. If not provided, the uploader will compute it",
461
- },
462
- "filesize": {
463
- "type": ["number", "null"],
464
- "description": "File size",
465
- },
466
- "filetype": {
467
- "type": "string",
468
- "enum": [
469
- FileType.CAMM.value,
470
- FileType.GOPRO.value,
471
- FileType.BLACKVUE.value,
472
- FileType.VIDEO.value,
473
- ],
474
- "description": "The video file type",
475
- },
476
- },
477
- "required": [
478
- "filename",
479
- "filetype",
480
- ],
481
- },
482
- )
483
-
484
-
485
- ImageVideoDescriptionFileSchema = {
486
- "oneOf": [VideoDescriptionFileSchema, ImageDescriptionFileSchema]
487
- }
488
-
489
-
490
- def validate_image_desc(desc: T.Any) -> None:
491
- try:
492
- jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
493
- except jsonschema.ValidationError as ex:
494
- # do not use str(ex) which is more verbose
495
- raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
496
- try:
497
- map_capture_time_to_datetime(desc["MAPCaptureTime"])
498
- except ValueError as ex:
499
- raise exceptions.MapillaryMetadataValidationError(str(ex)) from ex
500
-
501
-
502
- def validate_video_desc(desc: T.Any) -> None:
503
- try:
504
- jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
505
- except jsonschema.ValidationError as ex:
506
- # do not use str(ex) which is more verbose
507
- raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
508
-
509
-
510
- def datetime_to_map_capture_time(time: T.Union[datetime.datetime, int, float]) -> str:
511
- if isinstance(time, (float, int)):
512
- dt = datetime.datetime.fromtimestamp(time, datetime.timezone.utc)
513
- # otherwise it will be assumed to be in local time
514
- dt = dt.replace(tzinfo=datetime.timezone.utc)
515
- else:
516
- # otherwise it will be assumed to be in local time
517
- dt = time.astimezone(datetime.timezone.utc)
518
- return datetime.datetime.strftime(dt, "%Y_%m_%d_%H_%M_%S_%f")[:-3]
519
-
520
-
521
- def map_capture_time_to_datetime(time: str) -> datetime.datetime:
522
- dt = datetime.datetime.strptime(time, "%Y_%m_%d_%H_%M_%S_%f")
523
- dt = dt.replace(tzinfo=datetime.timezone.utc)
524
- return dt
525
-
526
-
527
- @T.overload
528
- def as_desc(metadata: ImageMetadata) -> ImageDescription: ...
529
-
530
-
531
- @T.overload
532
- def as_desc(metadata: ErrorMetadata) -> ImageDescriptionError: ...
533
-
534
-
535
- @T.overload
536
- def as_desc(metadata: VideoMetadata) -> VideoDescription: ...
537
-
538
-
539
- def as_desc(metadata):
540
- if isinstance(metadata, ErrorMetadata):
541
- return _describe_error_desc(
542
- metadata.error, metadata.filename, metadata.filetype
543
- )
544
- elif isinstance(metadata, VideoMetadata):
545
- return _as_video_desc(metadata)
546
- else:
547
- assert isinstance(metadata, ImageMetadata)
548
- return _as_image_desc(metadata)
549
-
550
-
551
- def _as_video_desc(metadata: VideoMetadata) -> VideoDescription:
552
- desc: VideoDescription = {
553
- "filename": str(metadata.filename.resolve()),
554
- "md5sum": metadata.md5sum,
555
- "filetype": metadata.filetype.value,
556
- "filesize": metadata.filesize,
557
- "MAPGPSTrack": [_encode_point(p) for p in metadata.points],
558
- }
559
- if metadata.make:
560
- desc["MAPDeviceMake"] = metadata.make
561
- if metadata.model:
562
- desc["MAPDeviceModel"] = metadata.model
563
- return desc
564
-
565
-
566
- def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
567
- desc: ImageDescription = {
568
- "filename": str(metadata.filename.resolve()),
569
- "md5sum": metadata.md5sum,
570
- "filesize": metadata.filesize,
571
- "filetype": FileType.IMAGE.value,
572
- "MAPLatitude": round(metadata.lat, _COORDINATES_PRECISION),
573
- "MAPLongitude": round(metadata.lon, _COORDINATES_PRECISION),
574
- "MAPCaptureTime": datetime_to_map_capture_time(metadata.time),
575
- }
576
- if metadata.alt is not None:
577
- desc["MAPAltitude"] = round(metadata.alt, _ALTITUDE_PRECISION)
578
- if metadata.angle is not None:
579
- desc["MAPCompassHeading"] = {
580
- "TrueHeading": round(metadata.angle, _ANGLE_PRECISION),
581
- "MagneticHeading": round(metadata.angle, _ANGLE_PRECISION),
582
- }
583
- fields = dataclasses.fields(metadata)
584
- for field in fields:
585
- if field.name.startswith("MAP"):
586
- value = getattr(metadata, field.name)
587
- if value is not None:
588
- # ignore error: TypedDict key must be a string literal;
589
- # expected one of ("MAPMetaTags", "MAPDeviceMake", "MAPDeviceModel", "MAPGPSAccuracyMeters", "MAPCameraUUID", ...)
590
- desc[field.name] = value # type: ignore
591
- return desc
592
-
593
-
594
- @T.overload
595
- def from_desc(metadata: ImageDescription) -> ImageMetadata: ...
596
-
597
-
598
- @T.overload
599
- def from_desc(metadata: VideoDescription) -> VideoMetadata: ...
600
-
601
-
602
- def from_desc(desc):
603
- assert "error" not in desc
604
- if desc["filetype"] == FileType.IMAGE.value:
605
- return _from_image_desc(desc)
606
- else:
607
- return _from_video_desc(desc)
608
-
609
-
610
- def _from_image_desc(desc) -> ImageMetadata:
611
- kwargs: T.Dict = {}
612
- for k, v in desc.items():
613
- if k not in [
614
- "filename",
615
- "md5sum",
616
- "filesize",
617
- "filetype",
618
- "MAPLatitude",
619
- "MAPLongitude",
620
- "MAPAltitude",
621
- "MAPCaptureTime",
622
- "MAPCompassHeading",
623
- ]:
624
- kwargs[k] = v
625
-
626
- return ImageMetadata(
627
- filename=Path(desc["filename"]),
628
- md5sum=desc.get("md5sum"),
629
- filesize=desc.get("filesize"),
630
- lat=desc["MAPLatitude"],
631
- lon=desc["MAPLongitude"],
632
- alt=desc.get("MAPAltitude"),
633
- time=geo.as_unix_time(map_capture_time_to_datetime(desc["MAPCaptureTime"])),
634
- angle=desc.get("MAPCompassHeading", {}).get("TrueHeading"),
635
- width=None,
636
- height=None,
637
- **kwargs,
638
- )
639
-
640
-
641
- def _encode_point(p: geo.Point) -> T.Sequence[T.Union[float, int, None]]:
642
- entry = [
643
- int(p.time * 1000),
644
- round(p.lon, _COORDINATES_PRECISION),
645
- round(p.lat, _COORDINATES_PRECISION),
646
- round(p.alt, _ALTITUDE_PRECISION) if p.alt is not None else None,
647
- round(p.angle, _ANGLE_PRECISION) if p.angle is not None else None,
648
- ]
649
- return entry
650
-
651
-
652
- def _decode_point(entry: T.Sequence[T.Any]) -> geo.Point:
653
- time_ms, lon, lat, alt, angle = entry
654
- return geo.Point(time=time_ms / 1000, lon=lon, lat=lat, alt=alt, angle=angle)
655
-
656
-
657
- def _from_video_desc(desc: VideoDescription) -> VideoMetadata:
658
- return VideoMetadata(
659
- filename=Path(desc["filename"]),
660
- md5sum=desc["md5sum"],
661
- filesize=desc["filesize"],
662
- filetype=FileType(desc["filetype"]),
663
- points=[_decode_point(entry) for entry in desc["MAPGPSTrack"]],
664
- make=desc.get("MAPDeviceMake"),
665
- model=desc.get("MAPDeviceModel"),
666
- )
667
-
668
-
669
- def validate_and_fail_desc(desc: DescriptionOrError) -> DescriptionOrError:
670
- if "error" in desc:
671
- return desc
672
-
673
- filetype = desc.get("filetype")
674
- try:
675
- if filetype == FileType.IMAGE.value:
676
- validate_image_desc(desc)
677
- else:
678
- validate_video_desc(desc)
679
- except exceptions.MapillaryMetadataValidationError as ex:
680
- return _describe_error_desc(
681
- ex,
682
- Path(desc["filename"]),
683
- filetype=FileType(filetype) if filetype else None,
684
- )
685
-
686
- if not os.path.isfile(desc["filename"]):
687
- return _describe_error_desc(
688
- exceptions.MapillaryMetadataValidationError(
689
- f"No such file {desc['filename']}"
690
- ),
691
- Path(desc["filename"]),
692
- filetype=FileType(filetype) if filetype else None,
693
- )
694
-
695
- return desc
696
-
697
-
698
- # Same as validate_and_fail_desc but for the metadata dataclass
699
- def validate_and_fail_metadata(metadata: MetadataOrError) -> MetadataOrError:
700
- if isinstance(metadata, ErrorMetadata):
701
- return metadata
702
-
703
- if isinstance(metadata, ImageMetadata):
704
- filetype = FileType.IMAGE
705
- validate = validate_image_desc
706
- else:
707
- assert isinstance(metadata, VideoMetadata)
708
- filetype = metadata.filetype
709
- validate = validate_video_desc
710
-
711
- try:
712
- validate(as_desc(metadata))
713
- except exceptions.MapillaryMetadataValidationError as ex:
714
- # rethrow because the original error is too verbose
715
- return describe_error_metadata(
716
- ex,
717
- metadata.filename,
718
- filetype=filetype,
719
- )
720
-
721
- if not metadata.filename.is_file():
722
- return describe_error_metadata(
723
- exceptions.MapillaryMetadataValidationError(
724
- f"No such file {metadata.filename}"
725
- ),
726
- metadata.filename,
727
- filetype=filetype,
728
- )
729
-
730
- return metadata
731
-
732
-
733
- def desc_file_to_exif(
734
- desc: ImageDescription,
735
- ) -> ImageDescription:
736
- not_needed = ["MAPSequenceUUID"]
737
- removed = {
738
- key: value
739
- for key, value in desc.items()
740
- if key.startswith("MAP") and key not in not_needed
741
- }
742
- return T.cast(ImageDescription, removed)
743
-
744
-
745
178
  def group_and_sort_images(
746
- metadatas: T.Sequence[ImageMetadata],
747
- ) -> T.Dict[str, T.List[ImageMetadata]]:
179
+ metadatas: T.Iterable[ImageMetadata],
180
+ ) -> dict[str, list[ImageMetadata]]:
748
181
  # group metadatas by uuid
749
- sequences_by_uuid: T.Dict[str, T.List[ImageMetadata]] = {}
182
+ sequences_by_uuid: dict[str, list[ImageMetadata]] = {}
750
183
  missing_sequence_uuid = str(uuid.uuid4())
751
184
  for metadata in metadatas:
752
185
  if metadata.MAPSequenceUUID is None:
@@ -773,7 +206,3 @@ def update_sequence_md5sum(sequence: T.Iterable[ImageMetadata]) -> str:
773
206
  assert isinstance(metadata.md5sum, str), "md5sum should be calculated"
774
207
  md5.update(metadata.md5sum.encode("utf-8"))
775
208
  return md5.hexdigest()
776
-
777
-
778
- if __name__ == "__main__":
779
- print(json.dumps(ImageVideoDescriptionFileSchema, indent=4))