mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +237 -16
  3. mapillary_tools/authenticate.py +325 -64
  4. mapillary_tools/{geotag/blackvue_parser.py → blackvue_parser.py} +74 -54
  5. mapillary_tools/camm/camm_builder.py +55 -97
  6. mapillary_tools/camm/camm_parser.py +429 -181
  7. mapillary_tools/commands/__main__.py +12 -6
  8. mapillary_tools/commands/authenticate.py +8 -1
  9. mapillary_tools/commands/process.py +27 -51
  10. mapillary_tools/commands/process_and_upload.py +19 -5
  11. mapillary_tools/commands/sample_video.py +2 -3
  12. mapillary_tools/commands/upload.py +18 -9
  13. mapillary_tools/commands/video_process_and_upload.py +19 -5
  14. mapillary_tools/config.py +31 -13
  15. mapillary_tools/constants.py +47 -6
  16. mapillary_tools/exceptions.py +34 -35
  17. mapillary_tools/exif_read.py +221 -116
  18. mapillary_tools/exif_write.py +7 -7
  19. mapillary_tools/exiftool_read.py +33 -42
  20. mapillary_tools/exiftool_read_video.py +46 -33
  21. mapillary_tools/exiftool_runner.py +77 -0
  22. mapillary_tools/ffmpeg.py +24 -23
  23. mapillary_tools/geo.py +144 -120
  24. mapillary_tools/geotag/base.py +147 -0
  25. mapillary_tools/geotag/factory.py +291 -0
  26. mapillary_tools/geotag/geotag_images_from_exif.py +14 -131
  27. mapillary_tools/geotag/geotag_images_from_exiftool.py +126 -82
  28. mapillary_tools/geotag/geotag_images_from_gpx.py +53 -118
  29. mapillary_tools/geotag/geotag_images_from_gpx_file.py +13 -126
  30. mapillary_tools/geotag/geotag_images_from_nmea_file.py +4 -5
  31. mapillary_tools/geotag/geotag_images_from_video.py +53 -51
  32. mapillary_tools/geotag/geotag_videos_from_exiftool.py +97 -0
  33. mapillary_tools/geotag/geotag_videos_from_gpx.py +39 -0
  34. mapillary_tools/geotag/geotag_videos_from_video.py +20 -185
  35. mapillary_tools/geotag/image_extractors/base.py +18 -0
  36. mapillary_tools/geotag/image_extractors/exif.py +60 -0
  37. mapillary_tools/geotag/image_extractors/exiftool.py +18 -0
  38. mapillary_tools/geotag/options.py +160 -0
  39. mapillary_tools/geotag/utils.py +52 -16
  40. mapillary_tools/geotag/video_extractors/base.py +18 -0
  41. mapillary_tools/geotag/video_extractors/exiftool.py +70 -0
  42. mapillary_tools/{video_data_extraction/extractors/gpx_parser.py → geotag/video_extractors/gpx.py} +57 -39
  43. mapillary_tools/geotag/video_extractors/native.py +157 -0
  44. mapillary_tools/{geotag → gpmf}/gpmf_parser.py +205 -182
  45. mapillary_tools/{geotag → gpmf}/gps_filter.py +5 -3
  46. mapillary_tools/history.py +7 -13
  47. mapillary_tools/mp4/construct_mp4_parser.py +9 -8
  48. mapillary_tools/mp4/io_utils.py +0 -1
  49. mapillary_tools/mp4/mp4_sample_parser.py +36 -28
  50. mapillary_tools/mp4/simple_mp4_builder.py +10 -9
  51. mapillary_tools/mp4/simple_mp4_parser.py +13 -22
  52. mapillary_tools/process_geotag_properties.py +155 -392
  53. mapillary_tools/process_sequence_properties.py +562 -208
  54. mapillary_tools/sample_video.py +13 -20
  55. mapillary_tools/telemetry.py +26 -13
  56. mapillary_tools/types.py +111 -58
  57. mapillary_tools/upload.py +316 -298
  58. mapillary_tools/upload_api_v4.py +55 -122
  59. mapillary_tools/uploader.py +396 -254
  60. mapillary_tools/utils.py +42 -18
  61. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/METADATA +3 -2
  62. mapillary_tools-0.14.0a2.dist-info/RECORD +72 -0
  63. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/WHEEL +1 -1
  64. mapillary_tools/geotag/__init__.py +0 -1
  65. mapillary_tools/geotag/geotag_from_generic.py +0 -22
  66. mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -93
  67. mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +0 -145
  68. mapillary_tools/video_data_extraction/cli_options.py +0 -22
  69. mapillary_tools/video_data_extraction/extract_video_data.py +0 -176
  70. mapillary_tools/video_data_extraction/extractors/base_parser.py +0 -75
  71. mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +0 -34
  72. mapillary_tools/video_data_extraction/extractors/camm_parser.py +0 -38
  73. mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +0 -71
  74. mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +0 -53
  75. mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +0 -52
  76. mapillary_tools/video_data_extraction/extractors/gopro_parser.py +0 -43
  77. mapillary_tools/video_data_extraction/extractors/nmea_parser.py +0 -24
  78. mapillary_tools/video_data_extraction/video_data_parser_factory.py +0 -39
  79. mapillary_tools-0.13.3a1.dist-info/RECORD +0 -75
  80. /mapillary_tools/{geotag → gpmf}/gpmf_gps_filter.py +0 -0
  81. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/entry_points.txt +0 -0
  82. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info/licenses}/LICENSE +0 -0
  83. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
+ from __future__ import annotations
2
+
1
3
  import dataclasses
2
4
  import datetime
3
5
  import io
4
6
  import itertools
5
- import pathlib
6
7
  import typing as T
7
8
 
8
9
  import construct as C
9
10
 
10
11
  from .. import telemetry
11
12
  from ..mp4.mp4_sample_parser import MovieBoxParser, Sample, TrackBoxParser
12
- from ..telemetry import GPSFix, GPSPoint
13
13
 
14
14
  """
15
15
  Parsing GPS from GPMF data format stored in GoPros. See the GPMF spec: https://github.com/gopro/gpmf-parser
@@ -39,7 +39,7 @@ class KLVDict(T.TypedDict):
39
39
  type: bytes
40
40
  structure_size: int
41
41
  repeat: int
42
- data: T.List[T.Any]
42
+ data: list[T.Any]
43
43
 
44
44
 
45
45
  GPMFSampleData: C.GreedyRange
@@ -130,11 +130,103 @@ GPMFSampleData = C.GreedyRange(KLV)
130
130
 
131
131
 
132
132
  @dataclasses.dataclass
133
- class TelemetryData:
134
- gps: T.List[GPSPoint]
135
- accl: T.List[telemetry.AccelerationData]
136
- gyro: T.List[telemetry.GyroscopeData]
137
- magn: T.List[telemetry.MagnetometerData]
133
+ class GoProInfo:
134
+ # None indicates the data has been extracted,
135
+ # while [] indicates extracetd but no data point found
136
+ gps: list[telemetry.GPSPoint] | None = None
137
+ accl: list[telemetry.AccelerationData] | None = None
138
+ gyro: list[telemetry.GyroscopeData] | None = None
139
+ magn: list[telemetry.MagnetometerData] | None = None
140
+ make: str = "GoPro"
141
+ model: str = ""
142
+
143
+
144
+ def extract_gopro_info(
145
+ fp: T.BinaryIO, telemetry_only: bool = False
146
+ ) -> GoProInfo | None:
147
+ """
148
+ Return the GoProInfo object if found. None indicates it's not a valid GoPro video.
149
+ """
150
+
151
+ moov = MovieBoxParser.parse_stream(fp)
152
+ for track in moov.extract_tracks():
153
+ if _contains_gpmd_description(track):
154
+ gpmd_samples = _filter_gpmd_samples(track)
155
+
156
+ if telemetry_only:
157
+ points_by_dvid: dict[int, list[telemetry.GPSPoint]] | None = None
158
+ dvnm_by_dvid: dict[int, bytes] | None = None
159
+ accls_by_dvid: dict[int, list[telemetry.AccelerationData]] | None = {}
160
+ gyros_by_dvid: dict[int, list[telemetry.GyroscopeData]] | None = {}
161
+ magns_by_dvid: dict[int, list[telemetry.MagnetometerData]] | None = {}
162
+ else:
163
+ points_by_dvid = {}
164
+ dvnm_by_dvid = {}
165
+ accls_by_dvid = None
166
+ gyros_by_dvid = None
167
+ magns_by_dvid = None
168
+
169
+ device_found = _load_telemetry_from_samples(
170
+ fp,
171
+ gpmd_samples,
172
+ points_by_dvid=points_by_dvid,
173
+ accls_by_dvid=accls_by_dvid,
174
+ gyros_by_dvid=gyros_by_dvid,
175
+ magns_by_dvid=magns_by_dvid,
176
+ dvnm_by_dvid=dvnm_by_dvid,
177
+ )
178
+ # If no device found, it's likely some other cameras using
179
+ # the "gpmd" container format, e.g. VANTRUE N2S 4K Dashcam
180
+ if not device_found:
181
+ return None
182
+
183
+ gopro_info = GoProInfo()
184
+
185
+ if points_by_dvid is not None:
186
+ gps_points = list(points_by_dvid.values())[0] if points_by_dvid else []
187
+ # backfill forward from the first point with epoch time
188
+ _backfill_gps_timestamps(gps_points)
189
+ # backfill backward from the first point with epoch time in reversed order
190
+ _backfill_gps_timestamps(reversed(gps_points))
191
+ gopro_info.gps = gps_points
192
+
193
+ if accls_by_dvid is not None:
194
+ gopro_info.accl = (
195
+ list(accls_by_dvid.values())[0] if accls_by_dvid else []
196
+ )
197
+
198
+ if gyros_by_dvid is not None:
199
+ gopro_info.gyro = (
200
+ list(gyros_by_dvid.values())[0] if gyros_by_dvid else []
201
+ )
202
+
203
+ if magns_by_dvid is not None:
204
+ gopro_info.magn = (
205
+ list(magns_by_dvid.values())[0] if magns_by_dvid else []
206
+ )
207
+
208
+ if dvnm_by_dvid is not None:
209
+ gopro_info.model = _extract_camera_model_from_devices(dvnm_by_dvid)
210
+
211
+ return gopro_info
212
+
213
+ return None
214
+
215
+
216
+ def extract_camera_model(fp: T.BinaryIO) -> str:
217
+ moov = MovieBoxParser.parse_stream(fp)
218
+ for track in moov.extract_tracks():
219
+ if _contains_gpmd_description(track):
220
+ gpmd_samples = _filter_gpmd_samples(track)
221
+ dvnm_by_dvid: dict[int, bytes] = {}
222
+ device_found = _load_telemetry_from_samples(
223
+ fp, gpmd_samples, dvnm_by_dvid=dvnm_by_dvid
224
+ )
225
+ if not device_found:
226
+ return ""
227
+ return _extract_camera_model_from_devices(dvnm_by_dvid)
228
+
229
+ return ""
138
230
 
139
231
 
140
232
  def _gps5_timestamp_to_epoch_time(dtstr: str):
@@ -181,10 +273,10 @@ def _gps5_timestamp_to_epoch_time(dtstr: str):
181
273
  # [378081666, -1224280064, 9621, 1492, 138],
182
274
  # [378081662, -1224280049, 9592, 1476, 150],
183
275
  # ]
184
- def gps5_from_stream(
276
+ def _gps5_from_stream(
185
277
  stream: T.Sequence[KLVDict],
186
- ) -> T.Generator[GPSPoint, None, None]:
187
- indexed: T.Dict[bytes, T.List[T.List[T.Any]]] = {
278
+ ) -> T.Generator[telemetry.GPSPoint, None, None]:
279
+ indexed: dict[bytes, list[list[T.Any]]] = {
188
280
  klv["key"]: klv["data"] for klv in stream
189
281
  }
190
282
 
@@ -201,7 +293,7 @@ def gps5_from_stream(
201
293
 
202
294
  gpsf = indexed.get(b"GPSF")
203
295
  if gpsf is not None:
204
- gpsf_value = GPSFix(gpsf[0][0])
296
+ gpsf_value = telemetry.GPSFix(gpsf[0][0])
205
297
  else:
206
298
  gpsf_value = None
207
299
 
@@ -225,7 +317,7 @@ def gps5_from_stream(
225
317
  lat, lon, alt, ground_speed, _speed_3d = [
226
318
  v / s for v, s in zip(point, scal_values)
227
319
  ]
228
- yield GPSPoint(
320
+ yield telemetry.GPSPoint(
229
321
  # will figure out the actual timestamp later
230
322
  time=0,
231
323
  lat=lat,
@@ -265,12 +357,12 @@ def _get_gps_type(input) -> bytes:
265
357
  return final
266
358
 
267
359
 
268
- def gps9_from_stream(
360
+ def _gps9_from_stream(
269
361
  stream: T.Sequence[KLVDict],
270
- ) -> T.Generator[GPSPoint, None, None]:
362
+ ) -> T.Generator[telemetry.GPSPoint, None, None]:
271
363
  NUM_VALUES = 9
272
364
 
273
- indexed: T.Dict[bytes, T.List[T.List[T.Any]]] = {
365
+ indexed: dict[bytes, list[list[T.Any]]] = {
274
366
  klv["key"]: klv["data"] for klv in stream
275
367
  }
276
368
 
@@ -322,14 +414,14 @@ def gps9_from_stream(
322
414
 
323
415
  epoch_time = _gps9_timestamp_to_epoch_time(days_since_2000, secs_since_midnight)
324
416
 
325
- yield GPSPoint(
417
+ yield telemetry.GPSPoint(
326
418
  # will figure out the actual timestamp later
327
419
  time=0,
328
420
  lat=lat,
329
421
  lon=lon,
330
422
  alt=alt,
331
423
  epoch_time=epoch_time,
332
- fix=GPSFix(gps_fix),
424
+ fix=telemetry.GPSFix(gps_fix),
333
425
  precision=dop * 100,
334
426
  ground_speed=speed_2d,
335
427
  angle=None,
@@ -352,16 +444,16 @@ def _find_first_device_id(stream: T.Sequence[KLVDict]) -> int:
352
444
  return device_id
353
445
 
354
446
 
355
- def _find_first_gps_stream(stream: T.Sequence[KLVDict]) -> T.List[GPSPoint]:
356
- sample_points: T.List[GPSPoint] = []
447
+ def _find_first_gps_stream(stream: T.Sequence[KLVDict]) -> list[telemetry.GPSPoint]:
448
+ sample_points: list[telemetry.GPSPoint] = []
357
449
 
358
450
  for klv in stream:
359
451
  if klv["key"] == b"STRM":
360
- sample_points = list(gps9_from_stream(klv["data"]))
452
+ sample_points = list(_gps9_from_stream(klv["data"]))
361
453
  if sample_points:
362
454
  break
363
455
 
364
- sample_points = list(gps5_from_stream(klv["data"]))
456
+ sample_points = list(_gps5_from_stream(klv["data"]))
365
457
  if sample_points:
366
458
  break
367
459
 
@@ -377,7 +469,7 @@ def _is_matrix_calibration(matrix: T.Sequence[float]) -> bool:
377
469
 
378
470
 
379
471
  def _build_matrix(
380
- orin: T.Union[bytes, T.Sequence[int]], orio: T.Union[bytes, T.Sequence[int]]
472
+ orin: bytes | T.Sequence[int], orio: bytes | T.Sequence[int]
381
473
  ) -> T.Sequence[float]:
382
474
  matrix = []
383
475
 
@@ -411,14 +503,14 @@ def _apply_matrix(
411
503
  yield sum(matrix[row_start + x] * values[x] for x in range(size))
412
504
 
413
505
 
414
- def _flatten(nested: T.Sequence[T.Sequence[float]]) -> T.List[float]:
415
- output: T.List[float] = []
506
+ def _flatten(nested: T.Sequence[T.Sequence[float]]) -> list[float]:
507
+ output: list[float] = []
416
508
  for row in nested:
417
509
  output.extend(row)
418
510
  return output
419
511
 
420
512
 
421
- def _get_matrix(klv: T.Dict[bytes, KLVDict]) -> T.Optional[T.Sequence[float]]:
513
+ def _get_matrix(klv: dict[bytes, KLVDict]) -> T.Sequence[float] | None:
422
514
  mtrx = klv.get(b"MTRX")
423
515
  if mtrx is not None:
424
516
  matrix: T.Sequence[float] = _flatten(mtrx["data"])
@@ -438,7 +530,7 @@ def _get_matrix(klv: T.Dict[bytes, KLVDict]) -> T.Optional[T.Sequence[float]]:
438
530
  def _scale_and_calibrate(
439
531
  stream: T.Sequence[KLVDict], key: bytes
440
532
  ) -> T.Generator[T.Sequence[float], None, None]:
441
- indexed: T.Dict[bytes, KLVDict] = {klv["key"]: klv for klv in stream}
533
+ indexed: dict[bytes, KLVDict] = {klv["key"]: klv for klv in stream}
442
534
 
443
535
  klv = indexed.get(key)
444
536
  if klv is None:
@@ -469,7 +561,7 @@ def _scale_and_calibrate(
469
561
 
470
562
 
471
563
  def _find_first_telemetry_stream(stream: T.Sequence[KLVDict], key: bytes):
472
- values: T.List[T.Sequence[float]] = []
564
+ values: list[T.Sequence[float]] = []
473
565
 
474
566
  for klv in stream:
475
567
  if klv["key"] == b"STRM":
@@ -480,30 +572,7 @@ def _find_first_telemetry_stream(stream: T.Sequence[KLVDict], key: bytes):
480
572
  return values
481
573
 
482
574
 
483
- def _extract_dvnm_from_samples(
484
- fp: T.BinaryIO, samples: T.Iterable[Sample]
485
- ) -> T.Dict[int, bytes]:
486
- dvnm_by_dvid: T.Dict[int, bytes] = {}
487
-
488
- for sample in samples:
489
- fp.seek(sample.raw_sample.offset, io.SEEK_SET)
490
- data = fp.read(sample.raw_sample.size)
491
- gpmf_sample_data = T.cast(T.Dict, GPMFSampleData.parse(data))
492
-
493
- # iterate devices
494
- devices = (klv for klv in gpmf_sample_data if klv["key"] == b"DEVC")
495
- for device in devices:
496
- device_id = _find_first_device_id(device["data"])
497
- for klv in device["data"]:
498
- if klv["key"] == b"DVNM" and klv["data"]:
499
- # klv["data"] could be [b"H", b"e", b"r", b"o", b"8", b" ", b"B", b"l", b"a", b"c", b"k"]
500
- # or [b"Hero8 Black"]
501
- dvnm_by_dvid[device_id] = b"".join(klv["data"])
502
-
503
- return dvnm_by_dvid
504
-
505
-
506
- def _backfill_gps_timestamps(gps_points: T.Iterable[GPSPoint]) -> None:
575
+ def _backfill_gps_timestamps(gps_points: T.Iterable[telemetry.GPSPoint]) -> None:
507
576
  it = iter(gps_points)
508
577
 
509
578
  # find the first point with epoch time
@@ -525,94 +594,97 @@ def _backfill_gps_timestamps(gps_points: T.Iterable[GPSPoint]) -> None:
525
594
  last = point
526
595
 
527
596
 
528
- def _extract_points_from_samples(
529
- fp: T.BinaryIO, samples: T.Iterable[Sample]
530
- ) -> TelemetryData:
531
- # To keep GPS points from different devices separated
532
- points_by_dvid: T.Dict[int, T.List[GPSPoint]] = {}
533
- accls_by_dvid: T.Dict[int, T.List[telemetry.AccelerationData]] = {}
534
- gyros_by_dvid: T.Dict[int, T.List[telemetry.GyroscopeData]] = {}
535
- magns_by_dvid: T.Dict[int, T.List[telemetry.MagnetometerData]] = {}
597
+ # This API is designed for performance
598
+ def _load_telemetry_from_samples(
599
+ fp: T.BinaryIO,
600
+ samples: T.Iterable[Sample],
601
+ points_by_dvid: dict[int, list[telemetry.GPSPoint]] | None = None,
602
+ accls_by_dvid: dict[int, list[telemetry.AccelerationData]] | None = None,
603
+ gyros_by_dvid: dict[int, list[telemetry.GyroscopeData]] | None = None,
604
+ magns_by_dvid: dict[int, list[telemetry.MagnetometerData]] | None = None,
605
+ dvnm_by_dvid: dict[int, bytes] | None = None,
606
+ ) -> bool:
607
+ device_found: bool = False
536
608
 
537
- for sample in samples:
538
- fp.seek(sample.raw_sample.offset, io.SEEK_SET)
539
- data = fp.read(sample.raw_sample.size)
540
- gpmf_sample_data = T.cast(T.Dict, GPMFSampleData.parse(data))
609
+ for sample, sample_data in _iterate_read_sample_data(fp, samples):
610
+ try:
611
+ gpmf_sample_data = T.cast(T.Dict, GPMFSampleData.parse(sample_data))
612
+ except C.ConstructError:
613
+ continue
541
614
 
542
615
  # iterate devices
543
616
  devices = (klv for klv in gpmf_sample_data if klv["key"] == b"DEVC")
544
617
  for device in devices:
618
+ device_found = True
545
619
  device_id = _find_first_device_id(device["data"])
546
620
 
547
- sample_points = _find_first_gps_stream(device["data"])
548
- if sample_points:
549
- # interpolate timestamps in between
550
- avg_timedelta = sample.exact_timedelta / len(sample_points)
551
- for idx, point in enumerate(sample_points):
552
- point.time = sample.exact_time + avg_timedelta * idx
553
-
554
- device_points = points_by_dvid.setdefault(device_id, [])
555
- device_points.extend(sample_points)
556
-
557
- sample_accls = _find_first_telemetry_stream(device["data"], b"ACCL")
558
- if sample_accls:
559
- # interpolate timestamps in between
560
- avg_delta = sample.exact_timedelta / len(sample_accls)
561
- accls_by_dvid.setdefault(device_id, []).extend(
562
- telemetry.AccelerationData(
563
- time=sample.exact_time + avg_delta * idx,
564
- x=x,
565
- y=y,
566
- z=z,
621
+ if dvnm_by_dvid is not None:
622
+ for klv in device["data"]:
623
+ if klv["key"] == b"DVNM" and klv["data"]:
624
+ # klv["data"] could be [b"H", b"e", b"r", b"o", b"8", b" ", b"B", b"l", b"a", b"c", b"k"]
625
+ # or [b"Hero8 Black"]
626
+ dvnm_by_dvid[device_id] = b"".join(klv["data"])
627
+
628
+ if points_by_dvid is not None:
629
+ sample_points = _find_first_gps_stream(device["data"])
630
+ if sample_points:
631
+ # interpolate timestamps in between
632
+ avg_timedelta = sample.exact_timedelta / len(sample_points)
633
+ for idx, point in enumerate(sample_points):
634
+ point.time = sample.exact_time + avg_timedelta * idx
635
+
636
+ device_points = points_by_dvid.setdefault(device_id, [])
637
+ device_points.extend(sample_points)
638
+
639
+ if accls_by_dvid is not None:
640
+ sample_accls = _find_first_telemetry_stream(device["data"], b"ACCL")
641
+ if sample_accls:
642
+ # interpolate timestamps in between
643
+ avg_delta = sample.exact_timedelta / len(sample_accls)
644
+ accls_by_dvid.setdefault(device_id, []).extend(
645
+ telemetry.AccelerationData(
646
+ time=sample.exact_time + avg_delta * idx,
647
+ x=x,
648
+ y=y,
649
+ z=z,
650
+ )
651
+ for idx, (z, x, y, *_) in enumerate(sample_accls)
567
652
  )
568
- for idx, (z, x, y, *_) in enumerate(sample_accls)
569
- )
570
653
 
571
- sample_gyros = _find_first_telemetry_stream(device["data"], b"GYRO")
572
- if sample_gyros:
573
- # interpolate timestamps in between
574
- avg_delta = sample.exact_timedelta / len(sample_gyros)
575
- gyros_by_dvid.setdefault(device_id, []).extend(
576
- telemetry.GyroscopeData(
577
- time=sample.exact_time + avg_delta * idx,
578
- x=x,
579
- y=y,
580
- z=z,
654
+ if gyros_by_dvid is not None:
655
+ sample_gyros = _find_first_telemetry_stream(device["data"], b"GYRO")
656
+ if sample_gyros:
657
+ # interpolate timestamps in between
658
+ avg_delta = sample.exact_timedelta / len(sample_gyros)
659
+ gyros_by_dvid.setdefault(device_id, []).extend(
660
+ telemetry.GyroscopeData(
661
+ time=sample.exact_time + avg_delta * idx,
662
+ x=x,
663
+ y=y,
664
+ z=z,
665
+ )
666
+ for idx, (z, x, y, *_) in enumerate(sample_gyros)
581
667
  )
582
- for idx, (z, x, y, *_) in enumerate(sample_gyros)
583
- )
584
668
 
585
- sample_magns = _find_first_telemetry_stream(device["data"], b"MAGN")
586
- if sample_magns:
587
- # interpolate timestamps in between
588
- avg_delta = sample.exact_timedelta / len(sample_magns)
589
- magns_by_dvid.setdefault(device_id, []).extend(
590
- telemetry.MagnetometerData(
591
- time=sample.exact_time + avg_delta * idx,
592
- x=x,
593
- y=y,
594
- z=z,
669
+ if magns_by_dvid is not None:
670
+ sample_magns = _find_first_telemetry_stream(device["data"], b"MAGN")
671
+ if sample_magns:
672
+ # interpolate timestamps in between
673
+ avg_delta = sample.exact_timedelta / len(sample_magns)
674
+ magns_by_dvid.setdefault(device_id, []).extend(
675
+ telemetry.MagnetometerData(
676
+ time=sample.exact_time + avg_delta * idx,
677
+ x=x,
678
+ y=y,
679
+ z=z,
680
+ )
681
+ for idx, (z, x, y, *_) in enumerate(sample_magns)
595
682
  )
596
- for idx, (z, x, y, *_) in enumerate(sample_magns)
597
- )
598
-
599
- gps_points = list(points_by_dvid.values())[0] if points_by_dvid else []
600
-
601
- # backfill forward from the first point with epoch time
602
- _backfill_gps_timestamps(gps_points)
603
683
 
604
- # backfill backward from the first point with epoch time in reversed order
605
- _backfill_gps_timestamps(reversed(gps_points))
606
-
607
- return TelemetryData(
608
- gps=gps_points,
609
- accl=list(accls_by_dvid.values())[0] if accls_by_dvid else [],
610
- gyro=list(gyros_by_dvid.values())[0] if gyros_by_dvid else [],
611
- magn=list(magns_by_dvid.values())[0] if magns_by_dvid else [],
612
- )
684
+ return device_found
613
685
 
614
686
 
615
- def _is_gpmd_description(description: T.Dict) -> bool:
687
+ def _is_gpmd_description(description: dict) -> bool:
616
688
  return description["format"] == b"gpmd"
617
689
 
618
690
 
@@ -627,60 +699,11 @@ def _filter_gpmd_samples(track: TrackBoxParser) -> T.Generator[Sample, None, Non
627
699
  yield sample
628
700
 
629
701
 
630
- def extract_points(fp: T.BinaryIO) -> T.List[GPSPoint]:
631
- """
632
- Return a list of points (could be empty) if it is a valid GoPro video,
633
- otherwise None
634
- """
635
- moov = MovieBoxParser.parse_stream(fp)
636
- for track in moov.extract_tracks():
637
- if _contains_gpmd_description(track):
638
- gpmd_samples = _filter_gpmd_samples(track)
639
- telemetry = _extract_points_from_samples(fp, gpmd_samples)
640
- # return the firstly found non-empty points
641
- if telemetry.gps:
642
- return telemetry.gps
643
-
644
- # points could be empty list or None here
645
- return []
646
-
647
-
648
- def extract_telemetry_data(fp: T.BinaryIO) -> T.Optional[TelemetryData]:
649
- """
650
- Return the telemetry data from the first found GoPro GPMF track
651
- """
652
- moov = MovieBoxParser.parse_stream(fp)
653
-
654
- for track in moov.extract_tracks():
655
- if _contains_gpmd_description(track):
656
- gpmd_samples = _filter_gpmd_samples(track)
657
- telemetry = _extract_points_from_samples(fp, gpmd_samples)
658
- # return the firstly found non-empty points
659
- if telemetry.gps:
660
- return telemetry
661
-
662
- # points could be empty list or None here
663
- return None
664
-
665
-
666
- def extract_all_device_names(fp: T.BinaryIO) -> T.Dict[int, bytes]:
667
- moov = MovieBoxParser.parse_stream(fp)
668
- for track in moov.extract_tracks():
669
- if _contains_gpmd_description(track):
670
- gpmd_samples = _filter_gpmd_samples(track)
671
- device_names = _extract_dvnm_from_samples(fp, gpmd_samples)
672
- if device_names:
673
- return device_names
674
- return {}
675
-
676
-
677
- def extract_camera_model(fp: T.BinaryIO) -> str:
678
- device_names = extract_all_device_names(fp)
679
-
702
+ def _extract_camera_model_from_devices(device_names: dict[int, bytes]) -> str:
680
703
  if not device_names:
681
704
  return ""
682
705
 
683
- unicode_names: T.List[str] = []
706
+ unicode_names: list[str] = []
684
707
  for name in device_names.values():
685
708
  try:
686
709
  unicode_names.append(name.decode("utf-8"))
@@ -705,9 +728,9 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
705
728
  return unicode_names[0].strip()
706
729
 
707
730
 
708
- def parse_gpx(path: pathlib.Path) -> T.List[GPSPoint]:
709
- with path.open("rb") as fp:
710
- points = extract_points(fp)
711
- if points is None:
712
- return []
713
- return points
731
+ def _iterate_read_sample_data(
732
+ fp: T.BinaryIO, samples: T.Iterable[Sample]
733
+ ) -> T.Generator[tuple[Sample, bytes], None, None]:
734
+ for sample in samples:
735
+ fp.seek(sample.raw_sample.offset, io.SEEK_SET)
736
+ yield (sample, fp.read(sample.raw_sample.size))
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import statistics
2
4
  import typing as T
3
5
 
@@ -96,7 +98,7 @@ def both(
96
98
  def dbscan(
97
99
  sequences: T.Sequence[PointSequence],
98
100
  merge_or_not: Decider,
99
- ) -> T.Dict[int, PointSequence]:
101
+ ) -> dict[int, PointSequence]:
100
102
  """
101
103
  One-dimension DBSCAN clustering: https://en.wikipedia.org/wiki/DBSCAN
102
104
  The input is a list of sequences, and it is guaranteed that all sequences are sorted by time.
@@ -107,7 +109,7 @@ def dbscan(
107
109
  """
108
110
 
109
111
  # find which sequences (keys) should be merged to which sequences (values)
110
- mergeto: T.Dict[int, int] = {}
112
+ mergeto: dict[int, int] = {}
111
113
  for left in range(len(sequences)):
112
114
  mergeto.setdefault(left, left)
113
115
  # find the first sequence to merge with
@@ -119,7 +121,7 @@ def dbscan(
119
121
  break
120
122
 
121
123
  # merge
122
- merged: T.Dict[int, PointSequence] = {}
124
+ merged: dict[int, PointSequence] = {}
123
125
  for idx, s in enumerate(sequences):
124
126
  merged.setdefault(mergeto[idx], []).extend(s)
125
127
 
@@ -1,6 +1,7 @@
1
+ from __future__ import annotations
2
+
1
3
  import json
2
4
  import logging
3
- import os
4
5
  import string
5
6
  import typing as T
6
7
  from pathlib import Path
@@ -10,13 +11,6 @@ from . import constants, types
10
11
  JSONDict = T.Dict[str, T.Union[str, int, float, None]]
11
12
 
12
13
  LOG = logging.getLogger(__name__)
13
- MAPILLARY_UPLOAD_HISTORY_PATH = os.getenv(
14
- "MAPILLARY_UPLOAD_HISTORY_PATH",
15
- os.path.join(
16
- constants.USER_DATA_DIR,
17
- "upload_history",
18
- ),
19
- )
20
14
 
21
15
 
22
16
  def _validate_hexdigits(md5sum: str):
@@ -35,14 +29,14 @@ def history_desc_path(md5sum: str) -> Path:
35
29
  basename = md5sum[2:]
36
30
  assert basename, f"Invalid md5sum {md5sum}"
37
31
  return (
38
- Path(MAPILLARY_UPLOAD_HISTORY_PATH)
32
+ Path(constants.MAPILLARY_UPLOAD_HISTORY_PATH)
39
33
  .joinpath(subfolder)
40
34
  .joinpath(f"{basename}.json")
41
35
  )
42
36
 
43
37
 
44
38
  def is_uploaded(md5sum: str) -> bool:
45
- if not MAPILLARY_UPLOAD_HISTORY_PATH:
39
+ if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
46
40
  return False
47
41
  return history_desc_path(md5sum).is_file()
48
42
 
@@ -51,14 +45,14 @@ def write_history(
51
45
  md5sum: str,
52
46
  params: JSONDict,
53
47
  summary: JSONDict,
54
- metadatas: T.Optional[T.Sequence[types.Metadata]] = None,
48
+ metadatas: T.Sequence[types.Metadata] | None = None,
55
49
  ) -> None:
56
- if not MAPILLARY_UPLOAD_HISTORY_PATH:
50
+ if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
57
51
  return
58
52
  path = history_desc_path(md5sum)
59
53
  LOG.debug("Writing upload history: %s", path)
60
54
  path.resolve().parent.mkdir(parents=True, exist_ok=True)
61
- history: T.Dict[str, T.Any] = {
55
+ history: dict[str, T.Any] = {
62
56
  "params": params,
63
57
  "summary": summary,
64
58
  }