mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +94 -4
  3. mapillary_tools/{geotag → camm}/camm_builder.py +73 -61
  4. mapillary_tools/camm/camm_parser.py +561 -0
  5. mapillary_tools/commands/__init__.py +0 -1
  6. mapillary_tools/commands/__main__.py +0 -6
  7. mapillary_tools/commands/process.py +0 -50
  8. mapillary_tools/commands/upload.py +1 -26
  9. mapillary_tools/constants.py +2 -2
  10. mapillary_tools/exiftool_read_video.py +13 -11
  11. mapillary_tools/ffmpeg.py +2 -2
  12. mapillary_tools/geo.py +0 -54
  13. mapillary_tools/geotag/blackvue_parser.py +4 -4
  14. mapillary_tools/geotag/geotag_images_from_exif.py +2 -1
  15. mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -1
  16. mapillary_tools/geotag/geotag_images_from_gpx_file.py +7 -1
  17. mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +5 -3
  18. mapillary_tools/geotag/geotag_videos_from_video.py +13 -14
  19. mapillary_tools/geotag/gpmf_gps_filter.py +9 -10
  20. mapillary_tools/geotag/gpmf_parser.py +346 -83
  21. mapillary_tools/mp4/__init__.py +0 -0
  22. mapillary_tools/{geotag → mp4}/construct_mp4_parser.py +32 -16
  23. mapillary_tools/mp4/mp4_sample_parser.py +322 -0
  24. mapillary_tools/{geotag → mp4}/simple_mp4_builder.py +64 -38
  25. mapillary_tools/process_geotag_properties.py +25 -19
  26. mapillary_tools/process_sequence_properties.py +6 -6
  27. mapillary_tools/sample_video.py +17 -16
  28. mapillary_tools/telemetry.py +71 -0
  29. mapillary_tools/types.py +18 -0
  30. mapillary_tools/upload.py +74 -233
  31. mapillary_tools/upload_api_v4.py +8 -9
  32. mapillary_tools/utils.py +9 -16
  33. mapillary_tools/video_data_extraction/cli_options.py +0 -1
  34. mapillary_tools/video_data_extraction/extract_video_data.py +13 -31
  35. mapillary_tools/video_data_extraction/extractors/base_parser.py +13 -11
  36. mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +5 -4
  37. mapillary_tools/video_data_extraction/extractors/camm_parser.py +13 -16
  38. mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +4 -9
  39. mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +9 -11
  40. mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +6 -11
  41. mapillary_tools/video_data_extraction/extractors/gopro_parser.py +11 -4
  42. mapillary_tools/video_data_extraction/extractors/gpx_parser.py +90 -11
  43. mapillary_tools/video_data_extraction/extractors/nmea_parser.py +3 -3
  44. mapillary_tools/video_data_extraction/video_data_parser_factory.py +13 -20
  45. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/METADATA +10 -3
  46. mapillary_tools-0.13.1.dist-info/RECORD +75 -0
  47. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/WHEEL +1 -1
  48. mapillary_tools/commands/upload_blackvue.py +0 -33
  49. mapillary_tools/commands/upload_camm.py +0 -33
  50. mapillary_tools/commands/upload_zip.py +0 -33
  51. mapillary_tools/geotag/camm_parser.py +0 -306
  52. mapillary_tools/geotag/mp4_sample_parser.py +0 -426
  53. mapillary_tools/process_import_meta_properties.py +0 -76
  54. mapillary_tools-0.12.1.dist-info/RECORD +0 -77
  55. /mapillary_tools/{geotag → mp4}/io_utils.py +0 -0
  56. /mapillary_tools/{geotag → mp4}/simple_mp4_parser.py +0 -0
  57. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/LICENSE +0 -0
  58. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/entry_points.txt +0 -0
  59. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/top_level.txt +0 -0
@@ -9,7 +9,8 @@ from pathlib import Path
9
9
 
10
10
  from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils
11
11
  from .exif_write import ExifEdit
12
- from .geotag import geotag_videos_from_video, mp4_sample_parser
12
+ from .geotag import geotag_videos_from_video
13
+ from .mp4 import mp4_sample_parser
13
14
  from .process_geotag_properties import GeotagSource
14
15
 
15
16
  LOG = logging.getLogger(__name__)
@@ -117,9 +118,9 @@ def sample_video(
117
118
  start_time=video_start_time_dt,
118
119
  )
119
120
  else:
120
- assert (
121
- 0 < video_sample_interval
122
- ), "expect positive video_sample_interval but got {video_sample_interval}"
121
+ assert 0 < video_sample_interval, (
122
+ "expect positive video_sample_interval but got {video_sample_interval}"
123
+ )
123
124
  _sample_single_video_by_interval(
124
125
  video_path,
125
126
  sample_dir,
@@ -166,7 +167,7 @@ def wip_dir_context(wip_dir: Path, done_dir: Path, rename_timeout_sec: int = 10)
166
167
  except Exception as e:
167
168
  time.sleep(1)
168
169
  error = e
169
- if not renamed and not error is None:
170
+ if not renamed and error is not None:
170
171
  raise error
171
172
  else:
172
173
  wip_dir.rename(done_dir)
@@ -234,10 +235,10 @@ def _sample_video_stream_by_distance(
234
235
  """
235
236
 
236
237
  LOG.info("Extracting video samples")
237
- sorted_samples = list(video_track_parser.parse_samples())
238
+ sorted_samples = list(video_track_parser.extract_samples())
238
239
  # we need sort sampels by composition time (CT) not the decoding offset (DT)
239
240
  # CT is the oder of videos streaming to audiences, as well as the order ffmpeg sampling
240
- sorted_samples.sort(key=lambda sample: sample.composition_time_offset)
241
+ sorted_samples.sort(key=lambda sample: sample.exact_composition_time)
241
242
  LOG.info("Found total %d video samples", len(sorted_samples))
242
243
 
243
244
  # interpolate sample points between the GPS track range (with 1ms buffer)
@@ -251,11 +252,11 @@ def _sample_video_stream_by_distance(
251
252
  (
252
253
  frame_idx_0based,
253
254
  video_sample,
254
- interpolator.interpolate(video_sample.composition_time_offset),
255
+ interpolator.interpolate(video_sample.exact_composition_time),
255
256
  )
256
257
  for frame_idx_0based, video_sample in enumerate(sorted_samples)
257
258
  if _within_track_time_range_buffered(
258
- points, video_sample.composition_time_offset
259
+ points, video_sample.exact_composition_time
259
260
  )
260
261
  ]
261
262
  LOG.info("Found total %d interpolated video samples", len(interp_sample_points))
@@ -316,7 +317,7 @@ def _sample_single_video_by_distance(
316
317
  LOG.info("Extracting video samples")
317
318
  video_stream_idx = video_stream["index"]
318
319
  moov_parser = mp4_sample_parser.MovieBoxParser.parse_file(video_path)
319
- video_track_parser = moov_parser.parse_track_at(video_stream_idx)
320
+ video_track_parser = moov_parser.extract_track_at(video_stream_idx)
320
321
  sample_points_by_frame_idx = _sample_video_stream_by_distance(
321
322
  video_metadata.points, video_track_parser, sample_distance
322
323
  )
@@ -338,9 +339,9 @@ def _sample_single_video_by_distance(
338
339
  f"Expect {len(sorted_sample_indices)} samples but extracted {len(frame_samples)} samples"
339
340
  )
340
341
  for idx, (frame_idx_1based, sample_paths) in enumerate(frame_samples):
341
- assert (
342
- len(sample_paths) == 1
343
- ), "Expect 1 sample path at {frame_idx_1based} but got {sample_paths}"
342
+ assert len(sample_paths) == 1, (
343
+ "Expect 1 sample path at {frame_idx_1based} but got {sample_paths}"
344
+ )
344
345
  if idx + 1 != frame_idx_1based:
345
346
  raise exceptions.MapillaryVideoError(
346
347
  f"Expect {sample_paths[0]} to be {idx + 1}th sample but got {frame_idx_1based}"
@@ -351,9 +352,9 @@ def _sample_single_video_by_distance(
351
352
  continue
352
353
 
353
354
  video_sample, interp = sample_points_by_frame_idx[sample_idx]
354
- assert (
355
- interp.time == video_sample.composition_time_offset
356
- ), f"interpolated time {interp.time} should match the video sample time {video_sample.composition_time_offset}"
355
+ assert interp.time == video_sample.exact_composition_time, (
356
+ f"interpolated time {interp.time} should match the video sample time {video_sample.exact_composition_time}"
357
+ )
357
358
 
358
359
  timestamp = start_time + datetime.timedelta(seconds=interp.time)
359
360
  exif_edit = ExifEdit(sample_paths[0])
@@ -0,0 +1,71 @@
1
+ import dataclasses
2
+ import typing as T
3
+ from enum import Enum, unique
4
+
5
+ from .geo import Point
6
+
7
+
8
+ @unique
9
+ class GPSFix(Enum):
10
+ NO_FIX = 0
11
+ FIX_2D = 2
12
+ FIX_3D = 3
13
+
14
+
15
+ @dataclasses.dataclass(order=True)
16
+ class TelemetryMeasurement:
17
+ """Base class for all telemetry measurements.
18
+
19
+ All telemetry measurements must have a timestamp in seconds.
20
+ This is an abstract base class - do not instantiate directly.
21
+ Instead use the concrete subclasses: AccelerationData, GyroscopeData, etc.
22
+ """
23
+
24
+ time: float
25
+
26
+
27
+ @dataclasses.dataclass
28
+ class GPSPoint(TelemetryMeasurement, Point):
29
+ epoch_time: T.Optional[float]
30
+ fix: T.Optional[GPSFix]
31
+ precision: T.Optional[float]
32
+ ground_speed: T.Optional[float]
33
+
34
+
35
+ @dataclasses.dataclass
36
+ class CAMMGPSPoint(TelemetryMeasurement, Point):
37
+ time_gps_epoch: float
38
+ gps_fix_type: int
39
+ horizontal_accuracy: float
40
+ vertical_accuracy: float
41
+ velocity_east: float
42
+ velocity_north: float
43
+ velocity_up: float
44
+ speed_accuracy: float
45
+
46
+
47
+ @dataclasses.dataclass(order=True)
48
+ class GyroscopeData(TelemetryMeasurement):
49
+ """Gyroscope signal in radians/seconds around XYZ axes of the camera."""
50
+
51
+ x: float
52
+ y: float
53
+ z: float
54
+
55
+
56
+ @dataclasses.dataclass(order=True)
57
+ class AccelerationData(TelemetryMeasurement):
58
+ """Accelerometer reading in meters/second^2 along XYZ axes of the camera."""
59
+
60
+ x: float
61
+ y: float
62
+ z: float
63
+
64
+
65
+ @dataclasses.dataclass(order=True)
66
+ class MagnetometerData(TelemetryMeasurement):
67
+ """Ambient magnetic field."""
68
+
69
+ x: float
70
+ y: float
71
+ z: float
mapillary_tools/types.py CHANGED
@@ -36,6 +36,7 @@ class FileType(enum.Enum):
36
36
  GOPRO = "gopro"
37
37
  IMAGE = "image"
38
38
  VIDEO = "video"
39
+ ZIP = "zip"
39
40
 
40
41
 
41
42
  @dataclasses.dataclass
@@ -56,6 +57,7 @@ class ImageMetadata(geo.Point):
56
57
  MAPMetaTags: T.Optional[T.Dict] = None
57
58
  # deprecated since v0.10.0; keep here for compatibility
58
59
  MAPFilename: T.Optional[str] = None
60
+ filesize: T.Optional[int] = None
59
61
 
60
62
  def update_md5sum(self, image_data: T.Optional[T.BinaryIO] = None) -> None:
61
63
  if self.md5sum is None:
@@ -81,6 +83,7 @@ class VideoMetadata:
81
83
  points: T.Sequence[geo.Point]
82
84
  make: T.Optional[str] = None
83
85
  model: T.Optional[str] = None
86
+ filesize: T.Optional[int] = None
84
87
 
85
88
  def update_md5sum(self) -> None:
86
89
  if self.md5sum is None:
@@ -143,6 +146,7 @@ class ImageDescription(_SequenceOnly, _Image, MetaProperties, total=True):
143
146
  # if None or absent, it will be calculated
144
147
  md5sum: T.Optional[str]
145
148
  filetype: Literal["image"]
149
+ filesize: T.Optional[int]
146
150
 
147
151
 
148
152
  class _VideoDescriptionRequired(TypedDict, total=True):
@@ -156,6 +160,7 @@ class _VideoDescriptionRequired(TypedDict, total=True):
156
160
  class VideoDescription(_VideoDescriptionRequired, total=False):
157
161
  MAPDeviceMake: str
158
162
  MAPDeviceModel: str
163
+ filesize: T.Optional[int]
159
164
 
160
165
 
161
166
  class _ErrorDescription(TypedDict, total=False):
@@ -368,6 +373,10 @@ ImageDescriptionFileSchema = merge_schema(
368
373
  "type": ["string", "null"],
369
374
  "description": "MD5 checksum of the image content. If not provided, the uploader will compute it",
370
375
  },
376
+ "filesize": {
377
+ "type": ["number", "null"],
378
+ "description": "File size",
379
+ },
371
380
  "filetype": {
372
381
  "type": "string",
373
382
  "enum": [FileType.IMAGE.value],
@@ -394,6 +403,10 @@ VideoDescriptionFileSchema = merge_schema(
394
403
  "type": ["string", "null"],
395
404
  "description": "MD5 checksum of the video content. If not provided, the uploader will compute it",
396
405
  },
406
+ "filesize": {
407
+ "type": ["number", "null"],
408
+ "description": "File size",
409
+ },
397
410
  "filetype": {
398
411
  "type": "string",
399
412
  "enum": [
@@ -484,6 +497,7 @@ def _as_video_desc(metadata: VideoMetadata) -> VideoDescription:
484
497
  "filename": str(metadata.filename.resolve()),
485
498
  "md5sum": metadata.md5sum,
486
499
  "filetype": metadata.filetype.value,
500
+ "filesize": metadata.filesize,
487
501
  "MAPGPSTrack": [_encode_point(p) for p in metadata.points],
488
502
  }
489
503
  if metadata.make:
@@ -497,6 +511,7 @@ def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
497
511
  desc: ImageDescription = {
498
512
  "filename": str(metadata.filename.resolve()),
499
513
  "md5sum": metadata.md5sum,
514
+ "filesize": metadata.filesize,
500
515
  "filetype": FileType.IMAGE.value,
501
516
  "MAPLatitude": round(metadata.lat, _COORDINATES_PRECISION),
502
517
  "MAPLongitude": round(metadata.lon, _COORDINATES_PRECISION),
@@ -542,6 +557,7 @@ def _from_image_desc(desc) -> ImageMetadata:
542
557
  if k not in [
543
558
  "filename",
544
559
  "md5sum",
560
+ "filesize",
545
561
  "filetype",
546
562
  "MAPLatitude",
547
563
  "MAPLongitude",
@@ -554,6 +570,7 @@ def _from_image_desc(desc) -> ImageMetadata:
554
570
  return ImageMetadata(
555
571
  filename=Path(desc["filename"]),
556
572
  md5sum=desc.get("md5sum"),
573
+ filesize=desc.get("filesize"),
557
574
  lat=desc["MAPLatitude"],
558
575
  lon=desc["MAPLongitude"],
559
576
  alt=desc.get("MAPAltitude"),
@@ -585,6 +602,7 @@ def _from_video_desc(desc: VideoDescription) -> VideoMetadata:
585
602
  return VideoMetadata(
586
603
  filename=Path(desc["filename"]),
587
604
  md5sum=desc["md5sum"],
605
+ filesize=desc["filesize"],
588
606
  filetype=FileType(desc["filetype"]),
589
607
  points=[_decode_point(entry) for entry in desc["MAPGPSTrack"]],
590
608
  make=desc.get("MAPDeviceMake"),
mapillary_tools/upload.py CHANGED
@@ -1,4 +1,3 @@
1
- import enum
2
1
  import json
3
2
  import logging
4
3
  import os
@@ -17,7 +16,6 @@ from . import (
17
16
  config,
18
17
  constants,
19
18
  exceptions,
20
- geo,
21
19
  history,
22
20
  ipc,
23
21
  types,
@@ -26,13 +24,9 @@ from . import (
26
24
  utils,
27
25
  VERSION,
28
26
  )
29
- from .geotag import (
30
- blackvue_parser,
31
- camm_builder,
32
- camm_parser,
33
- simple_mp4_builder,
34
- utils as video_utils,
35
- )
27
+ from .camm import camm_builder, camm_parser
28
+ from .geotag import gpmf_parser
29
+ from .mp4 import simple_mp4_builder
36
30
  from .types import FileType
37
31
 
38
32
  JSONDict = T.Dict[str, T.Union[str, int, float, None]]
@@ -42,6 +36,7 @@ MAPILLARY_DISABLE_API_LOGGING = os.getenv("MAPILLARY_DISABLE_API_LOGGING")
42
36
  MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN = os.getenv(
43
37
  "MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"
44
38
  )
39
+ MAPILLARY__EXPERIMENTAL_ENABLE_IMU = os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU")
45
40
  CAMM_CONVERTABLES = {FileType.CAMM, FileType.BLACKVUE, FileType.GOPRO}
46
41
 
47
42
 
@@ -55,12 +50,6 @@ class UploadHTTPError(Exception):
55
50
  pass
56
51
 
57
52
 
58
- class DirectUploadFileType(enum.Enum):
59
- RAW_BLACKVUE = "raw_blackvue"
60
- RAW_CAMM = "raw_camm"
61
- ZIP = "zip"
62
-
63
-
64
53
  def wrap_http_exception(ex: requests.HTTPError):
65
54
  req = ex.request
66
55
  resp = ex.response
@@ -440,7 +429,7 @@ def _api_logging_finished(summary: T.Dict):
440
429
  action: api_v4.ActionType = "upload_finished_upload"
441
430
  LOG.debug("API Logging for action %s: %s", action, summary)
442
431
  try:
443
- api_v4.logging(
432
+ api_v4.log_event(
444
433
  action,
445
434
  summary,
446
435
  )
@@ -462,7 +451,7 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
462
451
  action: api_v4.ActionType = "upload_failed_upload"
463
452
  LOG.debug("API Logging for action %s: %s", action, payload)
464
453
  try:
465
- api_v4.logging(
454
+ api_v4.log_event(
466
455
  action,
467
456
  payload_with_reason,
468
457
  )
@@ -521,7 +510,6 @@ def _find_metadata_with_filename_existed_in(
521
510
 
522
511
  def upload(
523
512
  import_path: T.Union[Path, T.Sequence[Path]],
524
- filetypes: T.Set[T.Union[FileType, DirectUploadFileType]],
525
513
  desc_path: T.Optional[str] = None,
526
514
  _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None,
527
515
  user_name: T.Optional[str] = None,
@@ -529,19 +517,6 @@ def upload(
529
517
  dry_run=False,
530
518
  skip_subfolders=False,
531
519
  ) -> None:
532
- if (
533
- DirectUploadFileType.RAW_BLACKVUE in filetypes
534
- and FileType.BLACKVUE in filetypes
535
- ):
536
- raise exceptions.MapillaryBadParameterError(
537
- f"filetypes should contain either {DirectUploadFileType.RAW_BLACKVUE.value} or {FileType.BLACKVUE.value}, not both",
538
- )
539
-
540
- if DirectUploadFileType.RAW_CAMM in filetypes and FileType.CAMM in filetypes:
541
- raise exceptions.MapillaryBadParameterError(
542
- f"File types should contain either {DirectUploadFileType.RAW_CAMM.value} or {FileType.CAMM.value}, not both",
543
- )
544
-
545
520
  import_paths: T.Sequence[Path]
546
521
  if isinstance(import_path, Path):
547
522
  import_paths = [import_path]
@@ -560,14 +535,7 @@ def upload(
560
535
  f"Import file or directory not found: {path}"
561
536
  )
562
537
 
563
- if {
564
- DirectUploadFileType.RAW_CAMM,
565
- DirectUploadFileType.RAW_BLACKVUE,
566
- DirectUploadFileType.ZIP,
567
- }.issuperset(filetypes):
568
- metadatas = None
569
- else:
570
- metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
538
+ metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
571
539
 
572
540
  user_items = fetch_user_items(user_name, organization_key)
573
541
 
@@ -610,101 +578,79 @@ def upload(
610
578
  chunk_size=int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
611
579
  )
612
580
 
613
- # if more than one filetypes speficied, check filename suffixes,
614
- # i.e. files not ended with .jpg or .mp4 will be ignored
615
- check_file_suffix = len(filetypes) > 1
616
-
617
581
  try:
618
- if FileType.IMAGE in filetypes:
619
- image_paths = utils.find_images(
620
- import_paths,
621
- skip_subfolders=skip_subfolders,
622
- check_file_suffix=check_file_suffix,
623
- )
624
- # find descs that match the image paths from the import paths
625
- image_metadatas = [
626
- metadata
627
- for metadata in (metadatas or [])
628
- if isinstance(metadata, types.ImageMetadata)
629
- ]
630
- specified_image_metadatas = _find_metadata_with_filename_existed_in(
631
- image_metadatas, image_paths
582
+ image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
583
+ # find descs that match the image paths from the import paths
584
+ image_metadatas = [
585
+ metadata
586
+ for metadata in (metadatas or [])
587
+ if isinstance(metadata, types.ImageMetadata)
588
+ ]
589
+ specified_image_metadatas = _find_metadata_with_filename_existed_in(
590
+ image_metadatas, image_paths
591
+ )
592
+ if specified_image_metadatas:
593
+ try:
594
+ clusters = mly_uploader.upload_images(
595
+ specified_image_metadatas,
596
+ event_payload={"file_type": FileType.IMAGE.value},
597
+ )
598
+ except Exception as ex:
599
+ raise UploadError(ex) from ex
600
+
601
+ if clusters:
602
+ LOG.debug("Uploaded to cluster: %s", clusters)
603
+
604
+ video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
605
+ video_metadatas = [
606
+ metadata
607
+ for metadata in (metadatas or [])
608
+ if isinstance(metadata, types.VideoMetadata)
609
+ ]
610
+ specified_video_metadatas = _find_metadata_with_filename_existed_in(
611
+ video_metadatas, video_paths
612
+ )
613
+ for idx, video_metadata in enumerate(specified_video_metadatas):
614
+ video_metadata.update_md5sum()
615
+ assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
616
+
617
+ # extract telemetry measurements from GoPro videos
618
+ telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = []
619
+ if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
620
+ if video_metadata.filetype is FileType.GOPRO:
621
+ with video_metadata.filename.open("rb") as fp:
622
+ telemetry_data = gpmf_parser.extract_telemetry_data(fp)
623
+ if telemetry_data:
624
+ telemetry_measurements.extend(telemetry_data.accl)
625
+ telemetry_measurements.extend(telemetry_data.gyro)
626
+ telemetry_measurements.extend(telemetry_data.magn)
627
+ telemetry_measurements.sort(key=lambda m: m.time)
628
+
629
+ generator = camm_builder.camm_sample_generator2(
630
+ video_metadata, telemetry_measurements=telemetry_measurements
632
631
  )
633
- if specified_image_metadatas:
632
+
633
+ with video_metadata.filename.open("rb") as src_fp:
634
+ camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
635
+ event_payload: uploader.Progress = {
636
+ "total_sequence_count": len(specified_video_metadatas),
637
+ "sequence_idx": idx,
638
+ "file_type": video_metadata.filetype.value,
639
+ "import_path": str(video_metadata.filename),
640
+ }
634
641
  try:
635
- clusters = mly_uploader.upload_images(
636
- specified_image_metadatas,
637
- event_payload={"file_type": FileType.IMAGE.value},
642
+ cluster_id = mly_uploader.upload_stream(
643
+ T.cast(T.BinaryIO, camm_fp),
644
+ upload_api_v4.ClusterFileType.CAMM,
645
+ video_metadata.md5sum,
646
+ event_payload=event_payload,
638
647
  )
639
648
  except Exception as ex:
640
649
  raise UploadError(ex) from ex
650
+ LOG.debug("Uploaded to cluster: %s", cluster_id)
641
651
 
642
- if clusters:
643
- LOG.debug("Uploaded to cluster: %s", clusters)
644
-
645
- supported = CAMM_CONVERTABLES.intersection(filetypes)
646
- if supported:
647
- video_paths = utils.find_videos(
648
- import_paths,
649
- skip_subfolders=skip_subfolders,
650
- check_file_suffix=check_file_suffix,
651
- )
652
- video_metadatas = [
653
- metadata
654
- for metadata in (metadatas or [])
655
- if isinstance(metadata, types.VideoMetadata)
656
- ]
657
- specified_video_metadatas = _find_metadata_with_filename_existed_in(
658
- video_metadatas, video_paths
659
- )
660
- for idx, video_metadata in enumerate(specified_video_metadatas):
661
- video_metadata.update_md5sum()
662
- assert isinstance(
663
- video_metadata.md5sum, str
664
- ), "md5sum should be updated"
665
- generator = camm_builder.camm_sample_generator2(video_metadata)
666
- with video_metadata.filename.open("rb") as src_fp:
667
- camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
668
- event_payload: uploader.Progress = {
669
- "total_sequence_count": len(specified_video_metadatas),
670
- "sequence_idx": idx,
671
- "file_type": video_metadata.filetype.value,
672
- "import_path": str(video_metadata.filename),
673
- }
674
- try:
675
- cluster_id = mly_uploader.upload_stream(
676
- T.cast(T.BinaryIO, camm_fp),
677
- upload_api_v4.ClusterFileType.CAMM,
678
- video_metadata.md5sum,
679
- event_payload=event_payload,
680
- )
681
- except Exception as ex:
682
- raise UploadError(ex) from ex
683
- LOG.debug("Uploaded to cluster: %s", cluster_id)
684
-
685
- if DirectUploadFileType.RAW_BLACKVUE in filetypes:
686
- video_paths = utils.find_videos(
687
- import_paths,
688
- skip_subfolders=skip_subfolders,
689
- check_file_suffix=check_file_suffix,
690
- )
691
- _upload_raw_blackvues_DEPRECATED(mly_uploader, video_paths)
692
-
693
- if DirectUploadFileType.RAW_CAMM in filetypes:
694
- video_paths = utils.find_videos(
695
- import_paths,
696
- skip_subfolders=skip_subfolders,
697
- check_file_suffix=check_file_suffix,
698
- )
699
- _upload_raw_camm_DEPRECATED(mly_uploader, video_paths)
700
-
701
- if DirectUploadFileType.ZIP in filetypes:
702
- zip_paths = utils.find_zipfiles(
703
- import_paths,
704
- skip_subfolders=skip_subfolders,
705
- check_file_suffix=check_file_suffix,
706
- )
707
- _upload_zipfiles(mly_uploader, zip_paths)
652
+ zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
653
+ _upload_zipfiles(mly_uploader, zip_paths)
708
654
 
709
655
  except UploadError as ex:
710
656
  inner_ex = ex.inner_ex
@@ -743,111 +689,6 @@ def upload(
743
689
  LOG.info("Nothing uploaded. Bye.")
744
690
 
745
691
 
746
- def _check_blackvue_DEPRECATED(video_path: Path) -> None:
747
- # Skip in tests only because we don't have valid sample blackvue for tests
748
- if os.getenv("MAPILLARY__DISABLE_BLACKVUE_CHECK") == "YES":
749
- return
750
-
751
- points = blackvue_parser.parse_gps_points(video_path)
752
- if not points:
753
- raise exceptions.MapillaryGPXEmptyError("No GPS found in the BlackVue video")
754
-
755
- stationary = video_utils.is_video_stationary(
756
- geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
757
- )
758
- if stationary:
759
- raise exceptions.MapillaryStationaryVideoError("Stationary BlackVue video")
760
-
761
-
762
- def _upload_raw_blackvues_DEPRECATED(
763
- mly_uploader: uploader.Uploader,
764
- video_paths: T.Sequence[Path],
765
- ) -> None:
766
- for idx, video_path in enumerate(video_paths):
767
- event_payload: uploader.Progress = {
768
- "total_sequence_count": len(video_paths),
769
- "sequence_idx": idx,
770
- "file_type": DirectUploadFileType.RAW_BLACKVUE.value,
771
- "import_path": str(video_path),
772
- }
773
-
774
- try:
775
- _check_blackvue_DEPRECATED(video_path)
776
- except Exception as ex:
777
- LOG.warning(
778
- "Skipping %s %s due to: %s",
779
- DirectUploadFileType.RAW_BLACKVUE.value.upper(),
780
- video_path.name,
781
- ex,
782
- )
783
- continue
784
-
785
- with video_path.open("rb") as fp:
786
- upload_md5sum = utils.md5sum_fp(fp).hexdigest()
787
- try:
788
- cluster_id = mly_uploader.upload_stream(
789
- fp,
790
- upload_api_v4.ClusterFileType.BLACKVUE,
791
- upload_md5sum,
792
- event_payload=event_payload,
793
- )
794
- except Exception as ex:
795
- raise UploadError(ex) from ex
796
- LOG.debug("Uploaded to cluster: %s", cluster_id)
797
-
798
-
799
- def _check_camm_DEPRECATED(video_path: Path) -> None:
800
- # Skip in tests only because we don't have valid sample CAMM for tests
801
- if os.getenv("MAPILLARY__DISABLE_CAMM_CHECK") == "YES":
802
- return
803
-
804
- points = camm_parser.parse_gpx(video_path)
805
- if not points:
806
- raise exceptions.MapillaryGPXEmptyError("No GPS found in the CAMM video")
807
-
808
- stationary = video_utils.is_video_stationary(
809
- geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
810
- )
811
- if stationary:
812
- raise exceptions.MapillaryStationaryVideoError("Stationary CAMM video")
813
-
814
-
815
- def _upload_raw_camm_DEPRECATED(
816
- mly_uploader: uploader.Uploader,
817
- video_paths: T.Sequence[Path],
818
- ) -> None:
819
- for idx, video_path in enumerate(video_paths):
820
- event_payload: uploader.Progress = {
821
- "total_sequence_count": len(video_paths),
822
- "sequence_idx": idx,
823
- "file_type": DirectUploadFileType.RAW_CAMM.value,
824
- "import_path": str(video_path),
825
- }
826
- try:
827
- _check_camm_DEPRECATED(video_path)
828
- except Exception as ex:
829
- LOG.warning(
830
- "Skipping %s %s due to: %s",
831
- DirectUploadFileType.RAW_CAMM.value.upper(),
832
- video_path.name,
833
- ex,
834
- )
835
- continue
836
- try:
837
- with video_path.open("rb") as fp:
838
- upload_md5sum = utils.md5sum_fp(fp).hexdigest()
839
- cluster_id = mly_uploader.upload_stream(
840
- fp,
841
- upload_api_v4.ClusterFileType.CAMM,
842
- upload_md5sum,
843
- event_payload=event_payload,
844
- )
845
- except Exception as ex:
846
- raise UploadError(ex) from ex
847
-
848
- LOG.debug("Uploaded to cluster: %s", cluster_id)
849
-
850
-
851
692
  def _upload_zipfiles(
852
693
  mly_uploader: uploader.Uploader,
853
694
  zip_paths: T.Sequence[Path],
@@ -856,7 +697,7 @@ def _upload_zipfiles(
856
697
  event_payload: uploader.Progress = {
857
698
  "total_sequence_count": len(zip_paths),
858
699
  "sequence_idx": idx,
859
- "file_type": DirectUploadFileType.ZIP.value,
700
+ "file_type": FileType.ZIP.value,
860
701
  "import_path": str(zip_path),
861
702
  }
862
703
  try: