mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapillary_tools/__init__.py +1 -1
- mapillary_tools/api_v4.py +94 -4
- mapillary_tools/{geotag → camm}/camm_builder.py +73 -61
- mapillary_tools/camm/camm_parser.py +561 -0
- mapillary_tools/commands/__init__.py +0 -1
- mapillary_tools/commands/__main__.py +0 -6
- mapillary_tools/commands/process.py +0 -50
- mapillary_tools/commands/upload.py +1 -26
- mapillary_tools/constants.py +2 -2
- mapillary_tools/exiftool_read_video.py +13 -11
- mapillary_tools/ffmpeg.py +2 -2
- mapillary_tools/geo.py +0 -54
- mapillary_tools/geotag/blackvue_parser.py +4 -4
- mapillary_tools/geotag/geotag_images_from_exif.py +2 -1
- mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -1
- mapillary_tools/geotag/geotag_images_from_gpx_file.py +7 -1
- mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +5 -3
- mapillary_tools/geotag/geotag_videos_from_video.py +13 -14
- mapillary_tools/geotag/gpmf_gps_filter.py +9 -10
- mapillary_tools/geotag/gpmf_parser.py +346 -83
- mapillary_tools/mp4/__init__.py +0 -0
- mapillary_tools/{geotag → mp4}/construct_mp4_parser.py +32 -16
- mapillary_tools/mp4/mp4_sample_parser.py +322 -0
- mapillary_tools/{geotag → mp4}/simple_mp4_builder.py +64 -38
- mapillary_tools/process_geotag_properties.py +25 -19
- mapillary_tools/process_sequence_properties.py +6 -6
- mapillary_tools/sample_video.py +17 -16
- mapillary_tools/telemetry.py +71 -0
- mapillary_tools/types.py +18 -0
- mapillary_tools/upload.py +74 -233
- mapillary_tools/upload_api_v4.py +8 -9
- mapillary_tools/utils.py +9 -16
- mapillary_tools/video_data_extraction/cli_options.py +0 -1
- mapillary_tools/video_data_extraction/extract_video_data.py +13 -31
- mapillary_tools/video_data_extraction/extractors/base_parser.py +13 -11
- mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +5 -4
- mapillary_tools/video_data_extraction/extractors/camm_parser.py +13 -16
- mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +4 -9
- mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +9 -11
- mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +6 -11
- mapillary_tools/video_data_extraction/extractors/gopro_parser.py +11 -4
- mapillary_tools/video_data_extraction/extractors/gpx_parser.py +90 -11
- mapillary_tools/video_data_extraction/extractors/nmea_parser.py +3 -3
- mapillary_tools/video_data_extraction/video_data_parser_factory.py +13 -20
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/METADATA +10 -3
- mapillary_tools-0.13.1.dist-info/RECORD +75 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/WHEEL +1 -1
- mapillary_tools/commands/upload_blackvue.py +0 -33
- mapillary_tools/commands/upload_camm.py +0 -33
- mapillary_tools/commands/upload_zip.py +0 -33
- mapillary_tools/geotag/camm_parser.py +0 -306
- mapillary_tools/geotag/mp4_sample_parser.py +0 -426
- mapillary_tools/process_import_meta_properties.py +0 -76
- mapillary_tools-0.12.1.dist-info/RECORD +0 -77
- /mapillary_tools/{geotag → mp4}/io_utils.py +0 -0
- /mapillary_tools/{geotag → mp4}/simple_mp4_parser.py +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/LICENSE +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/entry_points.txt +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/top_level.txt +0 -0
mapillary_tools/sample_video.py
CHANGED
|
@@ -9,7 +9,8 @@ from pathlib import Path
|
|
|
9
9
|
|
|
10
10
|
from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils
|
|
11
11
|
from .exif_write import ExifEdit
|
|
12
|
-
from .geotag import geotag_videos_from_video
|
|
12
|
+
from .geotag import geotag_videos_from_video
|
|
13
|
+
from .mp4 import mp4_sample_parser
|
|
13
14
|
from .process_geotag_properties import GeotagSource
|
|
14
15
|
|
|
15
16
|
LOG = logging.getLogger(__name__)
|
|
@@ -117,9 +118,9 @@ def sample_video(
|
|
|
117
118
|
start_time=video_start_time_dt,
|
|
118
119
|
)
|
|
119
120
|
else:
|
|
120
|
-
assert (
|
|
121
|
-
|
|
122
|
-
)
|
|
121
|
+
assert 0 < video_sample_interval, (
|
|
122
|
+
"expect positive video_sample_interval but got {video_sample_interval}"
|
|
123
|
+
)
|
|
123
124
|
_sample_single_video_by_interval(
|
|
124
125
|
video_path,
|
|
125
126
|
sample_dir,
|
|
@@ -166,7 +167,7 @@ def wip_dir_context(wip_dir: Path, done_dir: Path, rename_timeout_sec: int = 10)
|
|
|
166
167
|
except Exception as e:
|
|
167
168
|
time.sleep(1)
|
|
168
169
|
error = e
|
|
169
|
-
if not renamed and
|
|
170
|
+
if not renamed and error is not None:
|
|
170
171
|
raise error
|
|
171
172
|
else:
|
|
172
173
|
wip_dir.rename(done_dir)
|
|
@@ -234,10 +235,10 @@ def _sample_video_stream_by_distance(
|
|
|
234
235
|
"""
|
|
235
236
|
|
|
236
237
|
LOG.info("Extracting video samples")
|
|
237
|
-
sorted_samples = list(video_track_parser.
|
|
238
|
+
sorted_samples = list(video_track_parser.extract_samples())
|
|
238
239
|
# we need sort sampels by composition time (CT) not the decoding offset (DT)
|
|
239
240
|
# CT is the oder of videos streaming to audiences, as well as the order ffmpeg sampling
|
|
240
|
-
sorted_samples.sort(key=lambda sample: sample.
|
|
241
|
+
sorted_samples.sort(key=lambda sample: sample.exact_composition_time)
|
|
241
242
|
LOG.info("Found total %d video samples", len(sorted_samples))
|
|
242
243
|
|
|
243
244
|
# interpolate sample points between the GPS track range (with 1ms buffer)
|
|
@@ -251,11 +252,11 @@ def _sample_video_stream_by_distance(
|
|
|
251
252
|
(
|
|
252
253
|
frame_idx_0based,
|
|
253
254
|
video_sample,
|
|
254
|
-
interpolator.interpolate(video_sample.
|
|
255
|
+
interpolator.interpolate(video_sample.exact_composition_time),
|
|
255
256
|
)
|
|
256
257
|
for frame_idx_0based, video_sample in enumerate(sorted_samples)
|
|
257
258
|
if _within_track_time_range_buffered(
|
|
258
|
-
points, video_sample.
|
|
259
|
+
points, video_sample.exact_composition_time
|
|
259
260
|
)
|
|
260
261
|
]
|
|
261
262
|
LOG.info("Found total %d interpolated video samples", len(interp_sample_points))
|
|
@@ -316,7 +317,7 @@ def _sample_single_video_by_distance(
|
|
|
316
317
|
LOG.info("Extracting video samples")
|
|
317
318
|
video_stream_idx = video_stream["index"]
|
|
318
319
|
moov_parser = mp4_sample_parser.MovieBoxParser.parse_file(video_path)
|
|
319
|
-
video_track_parser = moov_parser.
|
|
320
|
+
video_track_parser = moov_parser.extract_track_at(video_stream_idx)
|
|
320
321
|
sample_points_by_frame_idx = _sample_video_stream_by_distance(
|
|
321
322
|
video_metadata.points, video_track_parser, sample_distance
|
|
322
323
|
)
|
|
@@ -338,9 +339,9 @@ def _sample_single_video_by_distance(
|
|
|
338
339
|
f"Expect {len(sorted_sample_indices)} samples but extracted {len(frame_samples)} samples"
|
|
339
340
|
)
|
|
340
341
|
for idx, (frame_idx_1based, sample_paths) in enumerate(frame_samples):
|
|
341
|
-
assert (
|
|
342
|
-
|
|
343
|
-
)
|
|
342
|
+
assert len(sample_paths) == 1, (
|
|
343
|
+
"Expect 1 sample path at {frame_idx_1based} but got {sample_paths}"
|
|
344
|
+
)
|
|
344
345
|
if idx + 1 != frame_idx_1based:
|
|
345
346
|
raise exceptions.MapillaryVideoError(
|
|
346
347
|
f"Expect {sample_paths[0]} to be {idx + 1}th sample but got {frame_idx_1based}"
|
|
@@ -351,9 +352,9 @@ def _sample_single_video_by_distance(
|
|
|
351
352
|
continue
|
|
352
353
|
|
|
353
354
|
video_sample, interp = sample_points_by_frame_idx[sample_idx]
|
|
354
|
-
assert (
|
|
355
|
-
interp.time
|
|
356
|
-
)
|
|
355
|
+
assert interp.time == video_sample.exact_composition_time, (
|
|
356
|
+
f"interpolated time {interp.time} should match the video sample time {video_sample.exact_composition_time}"
|
|
357
|
+
)
|
|
357
358
|
|
|
358
359
|
timestamp = start_time + datetime.timedelta(seconds=interp.time)
|
|
359
360
|
exif_edit = ExifEdit(sample_paths[0])
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
import typing as T
|
|
3
|
+
from enum import Enum, unique
|
|
4
|
+
|
|
5
|
+
from .geo import Point
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@unique
|
|
9
|
+
class GPSFix(Enum):
|
|
10
|
+
NO_FIX = 0
|
|
11
|
+
FIX_2D = 2
|
|
12
|
+
FIX_3D = 3
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclasses.dataclass(order=True)
|
|
16
|
+
class TelemetryMeasurement:
|
|
17
|
+
"""Base class for all telemetry measurements.
|
|
18
|
+
|
|
19
|
+
All telemetry measurements must have a timestamp in seconds.
|
|
20
|
+
This is an abstract base class - do not instantiate directly.
|
|
21
|
+
Instead use the concrete subclasses: AccelerationData, GyroscopeData, etc.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
time: float
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclasses.dataclass
|
|
28
|
+
class GPSPoint(TelemetryMeasurement, Point):
|
|
29
|
+
epoch_time: T.Optional[float]
|
|
30
|
+
fix: T.Optional[GPSFix]
|
|
31
|
+
precision: T.Optional[float]
|
|
32
|
+
ground_speed: T.Optional[float]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclasses.dataclass
|
|
36
|
+
class CAMMGPSPoint(TelemetryMeasurement, Point):
|
|
37
|
+
time_gps_epoch: float
|
|
38
|
+
gps_fix_type: int
|
|
39
|
+
horizontal_accuracy: float
|
|
40
|
+
vertical_accuracy: float
|
|
41
|
+
velocity_east: float
|
|
42
|
+
velocity_north: float
|
|
43
|
+
velocity_up: float
|
|
44
|
+
speed_accuracy: float
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclasses.dataclass(order=True)
|
|
48
|
+
class GyroscopeData(TelemetryMeasurement):
|
|
49
|
+
"""Gyroscope signal in radians/seconds around XYZ axes of the camera."""
|
|
50
|
+
|
|
51
|
+
x: float
|
|
52
|
+
y: float
|
|
53
|
+
z: float
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclasses.dataclass(order=True)
|
|
57
|
+
class AccelerationData(TelemetryMeasurement):
|
|
58
|
+
"""Accelerometer reading in meters/second^2 along XYZ axes of the camera."""
|
|
59
|
+
|
|
60
|
+
x: float
|
|
61
|
+
y: float
|
|
62
|
+
z: float
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclasses.dataclass(order=True)
|
|
66
|
+
class MagnetometerData(TelemetryMeasurement):
|
|
67
|
+
"""Ambient magnetic field."""
|
|
68
|
+
|
|
69
|
+
x: float
|
|
70
|
+
y: float
|
|
71
|
+
z: float
|
mapillary_tools/types.py
CHANGED
|
@@ -36,6 +36,7 @@ class FileType(enum.Enum):
|
|
|
36
36
|
GOPRO = "gopro"
|
|
37
37
|
IMAGE = "image"
|
|
38
38
|
VIDEO = "video"
|
|
39
|
+
ZIP = "zip"
|
|
39
40
|
|
|
40
41
|
|
|
41
42
|
@dataclasses.dataclass
|
|
@@ -56,6 +57,7 @@ class ImageMetadata(geo.Point):
|
|
|
56
57
|
MAPMetaTags: T.Optional[T.Dict] = None
|
|
57
58
|
# deprecated since v0.10.0; keep here for compatibility
|
|
58
59
|
MAPFilename: T.Optional[str] = None
|
|
60
|
+
filesize: T.Optional[int] = None
|
|
59
61
|
|
|
60
62
|
def update_md5sum(self, image_data: T.Optional[T.BinaryIO] = None) -> None:
|
|
61
63
|
if self.md5sum is None:
|
|
@@ -81,6 +83,7 @@ class VideoMetadata:
|
|
|
81
83
|
points: T.Sequence[geo.Point]
|
|
82
84
|
make: T.Optional[str] = None
|
|
83
85
|
model: T.Optional[str] = None
|
|
86
|
+
filesize: T.Optional[int] = None
|
|
84
87
|
|
|
85
88
|
def update_md5sum(self) -> None:
|
|
86
89
|
if self.md5sum is None:
|
|
@@ -143,6 +146,7 @@ class ImageDescription(_SequenceOnly, _Image, MetaProperties, total=True):
|
|
|
143
146
|
# if None or absent, it will be calculated
|
|
144
147
|
md5sum: T.Optional[str]
|
|
145
148
|
filetype: Literal["image"]
|
|
149
|
+
filesize: T.Optional[int]
|
|
146
150
|
|
|
147
151
|
|
|
148
152
|
class _VideoDescriptionRequired(TypedDict, total=True):
|
|
@@ -156,6 +160,7 @@ class _VideoDescriptionRequired(TypedDict, total=True):
|
|
|
156
160
|
class VideoDescription(_VideoDescriptionRequired, total=False):
|
|
157
161
|
MAPDeviceMake: str
|
|
158
162
|
MAPDeviceModel: str
|
|
163
|
+
filesize: T.Optional[int]
|
|
159
164
|
|
|
160
165
|
|
|
161
166
|
class _ErrorDescription(TypedDict, total=False):
|
|
@@ -368,6 +373,10 @@ ImageDescriptionFileSchema = merge_schema(
|
|
|
368
373
|
"type": ["string", "null"],
|
|
369
374
|
"description": "MD5 checksum of the image content. If not provided, the uploader will compute it",
|
|
370
375
|
},
|
|
376
|
+
"filesize": {
|
|
377
|
+
"type": ["number", "null"],
|
|
378
|
+
"description": "File size",
|
|
379
|
+
},
|
|
371
380
|
"filetype": {
|
|
372
381
|
"type": "string",
|
|
373
382
|
"enum": [FileType.IMAGE.value],
|
|
@@ -394,6 +403,10 @@ VideoDescriptionFileSchema = merge_schema(
|
|
|
394
403
|
"type": ["string", "null"],
|
|
395
404
|
"description": "MD5 checksum of the video content. If not provided, the uploader will compute it",
|
|
396
405
|
},
|
|
406
|
+
"filesize": {
|
|
407
|
+
"type": ["number", "null"],
|
|
408
|
+
"description": "File size",
|
|
409
|
+
},
|
|
397
410
|
"filetype": {
|
|
398
411
|
"type": "string",
|
|
399
412
|
"enum": [
|
|
@@ -484,6 +497,7 @@ def _as_video_desc(metadata: VideoMetadata) -> VideoDescription:
|
|
|
484
497
|
"filename": str(metadata.filename.resolve()),
|
|
485
498
|
"md5sum": metadata.md5sum,
|
|
486
499
|
"filetype": metadata.filetype.value,
|
|
500
|
+
"filesize": metadata.filesize,
|
|
487
501
|
"MAPGPSTrack": [_encode_point(p) for p in metadata.points],
|
|
488
502
|
}
|
|
489
503
|
if metadata.make:
|
|
@@ -497,6 +511,7 @@ def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
|
|
|
497
511
|
desc: ImageDescription = {
|
|
498
512
|
"filename": str(metadata.filename.resolve()),
|
|
499
513
|
"md5sum": metadata.md5sum,
|
|
514
|
+
"filesize": metadata.filesize,
|
|
500
515
|
"filetype": FileType.IMAGE.value,
|
|
501
516
|
"MAPLatitude": round(metadata.lat, _COORDINATES_PRECISION),
|
|
502
517
|
"MAPLongitude": round(metadata.lon, _COORDINATES_PRECISION),
|
|
@@ -542,6 +557,7 @@ def _from_image_desc(desc) -> ImageMetadata:
|
|
|
542
557
|
if k not in [
|
|
543
558
|
"filename",
|
|
544
559
|
"md5sum",
|
|
560
|
+
"filesize",
|
|
545
561
|
"filetype",
|
|
546
562
|
"MAPLatitude",
|
|
547
563
|
"MAPLongitude",
|
|
@@ -554,6 +570,7 @@ def _from_image_desc(desc) -> ImageMetadata:
|
|
|
554
570
|
return ImageMetadata(
|
|
555
571
|
filename=Path(desc["filename"]),
|
|
556
572
|
md5sum=desc.get("md5sum"),
|
|
573
|
+
filesize=desc.get("filesize"),
|
|
557
574
|
lat=desc["MAPLatitude"],
|
|
558
575
|
lon=desc["MAPLongitude"],
|
|
559
576
|
alt=desc.get("MAPAltitude"),
|
|
@@ -585,6 +602,7 @@ def _from_video_desc(desc: VideoDescription) -> VideoMetadata:
|
|
|
585
602
|
return VideoMetadata(
|
|
586
603
|
filename=Path(desc["filename"]),
|
|
587
604
|
md5sum=desc["md5sum"],
|
|
605
|
+
filesize=desc["filesize"],
|
|
588
606
|
filetype=FileType(desc["filetype"]),
|
|
589
607
|
points=[_decode_point(entry) for entry in desc["MAPGPSTrack"]],
|
|
590
608
|
make=desc.get("MAPDeviceMake"),
|
mapillary_tools/upload.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import enum
|
|
2
1
|
import json
|
|
3
2
|
import logging
|
|
4
3
|
import os
|
|
@@ -17,7 +16,6 @@ from . import (
|
|
|
17
16
|
config,
|
|
18
17
|
constants,
|
|
19
18
|
exceptions,
|
|
20
|
-
geo,
|
|
21
19
|
history,
|
|
22
20
|
ipc,
|
|
23
21
|
types,
|
|
@@ -26,13 +24,9 @@ from . import (
|
|
|
26
24
|
utils,
|
|
27
25
|
VERSION,
|
|
28
26
|
)
|
|
29
|
-
from .
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
camm_parser,
|
|
33
|
-
simple_mp4_builder,
|
|
34
|
-
utils as video_utils,
|
|
35
|
-
)
|
|
27
|
+
from .camm import camm_builder, camm_parser
|
|
28
|
+
from .geotag import gpmf_parser
|
|
29
|
+
from .mp4 import simple_mp4_builder
|
|
36
30
|
from .types import FileType
|
|
37
31
|
|
|
38
32
|
JSONDict = T.Dict[str, T.Union[str, int, float, None]]
|
|
@@ -42,6 +36,7 @@ MAPILLARY_DISABLE_API_LOGGING = os.getenv("MAPILLARY_DISABLE_API_LOGGING")
|
|
|
42
36
|
MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN = os.getenv(
|
|
43
37
|
"MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"
|
|
44
38
|
)
|
|
39
|
+
MAPILLARY__EXPERIMENTAL_ENABLE_IMU = os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU")
|
|
45
40
|
CAMM_CONVERTABLES = {FileType.CAMM, FileType.BLACKVUE, FileType.GOPRO}
|
|
46
41
|
|
|
47
42
|
|
|
@@ -55,12 +50,6 @@ class UploadHTTPError(Exception):
|
|
|
55
50
|
pass
|
|
56
51
|
|
|
57
52
|
|
|
58
|
-
class DirectUploadFileType(enum.Enum):
|
|
59
|
-
RAW_BLACKVUE = "raw_blackvue"
|
|
60
|
-
RAW_CAMM = "raw_camm"
|
|
61
|
-
ZIP = "zip"
|
|
62
|
-
|
|
63
|
-
|
|
64
53
|
def wrap_http_exception(ex: requests.HTTPError):
|
|
65
54
|
req = ex.request
|
|
66
55
|
resp = ex.response
|
|
@@ -440,7 +429,7 @@ def _api_logging_finished(summary: T.Dict):
|
|
|
440
429
|
action: api_v4.ActionType = "upload_finished_upload"
|
|
441
430
|
LOG.debug("API Logging for action %s: %s", action, summary)
|
|
442
431
|
try:
|
|
443
|
-
api_v4.
|
|
432
|
+
api_v4.log_event(
|
|
444
433
|
action,
|
|
445
434
|
summary,
|
|
446
435
|
)
|
|
@@ -462,7 +451,7 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
|
|
|
462
451
|
action: api_v4.ActionType = "upload_failed_upload"
|
|
463
452
|
LOG.debug("API Logging for action %s: %s", action, payload)
|
|
464
453
|
try:
|
|
465
|
-
api_v4.
|
|
454
|
+
api_v4.log_event(
|
|
466
455
|
action,
|
|
467
456
|
payload_with_reason,
|
|
468
457
|
)
|
|
@@ -521,7 +510,6 @@ def _find_metadata_with_filename_existed_in(
|
|
|
521
510
|
|
|
522
511
|
def upload(
|
|
523
512
|
import_path: T.Union[Path, T.Sequence[Path]],
|
|
524
|
-
filetypes: T.Set[T.Union[FileType, DirectUploadFileType]],
|
|
525
513
|
desc_path: T.Optional[str] = None,
|
|
526
514
|
_metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None,
|
|
527
515
|
user_name: T.Optional[str] = None,
|
|
@@ -529,19 +517,6 @@ def upload(
|
|
|
529
517
|
dry_run=False,
|
|
530
518
|
skip_subfolders=False,
|
|
531
519
|
) -> None:
|
|
532
|
-
if (
|
|
533
|
-
DirectUploadFileType.RAW_BLACKVUE in filetypes
|
|
534
|
-
and FileType.BLACKVUE in filetypes
|
|
535
|
-
):
|
|
536
|
-
raise exceptions.MapillaryBadParameterError(
|
|
537
|
-
f"filetypes should contain either {DirectUploadFileType.RAW_BLACKVUE.value} or {FileType.BLACKVUE.value}, not both",
|
|
538
|
-
)
|
|
539
|
-
|
|
540
|
-
if DirectUploadFileType.RAW_CAMM in filetypes and FileType.CAMM in filetypes:
|
|
541
|
-
raise exceptions.MapillaryBadParameterError(
|
|
542
|
-
f"File types should contain either {DirectUploadFileType.RAW_CAMM.value} or {FileType.CAMM.value}, not both",
|
|
543
|
-
)
|
|
544
|
-
|
|
545
520
|
import_paths: T.Sequence[Path]
|
|
546
521
|
if isinstance(import_path, Path):
|
|
547
522
|
import_paths = [import_path]
|
|
@@ -560,14 +535,7 @@ def upload(
|
|
|
560
535
|
f"Import file or directory not found: {path}"
|
|
561
536
|
)
|
|
562
537
|
|
|
563
|
-
|
|
564
|
-
DirectUploadFileType.RAW_CAMM,
|
|
565
|
-
DirectUploadFileType.RAW_BLACKVUE,
|
|
566
|
-
DirectUploadFileType.ZIP,
|
|
567
|
-
}.issuperset(filetypes):
|
|
568
|
-
metadatas = None
|
|
569
|
-
else:
|
|
570
|
-
metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
|
|
538
|
+
metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
|
|
571
539
|
|
|
572
540
|
user_items = fetch_user_items(user_name, organization_key)
|
|
573
541
|
|
|
@@ -610,101 +578,79 @@ def upload(
|
|
|
610
578
|
chunk_size=int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
|
|
611
579
|
)
|
|
612
580
|
|
|
613
|
-
# if more than one filetypes speficied, check filename suffixes,
|
|
614
|
-
# i.e. files not ended with .jpg or .mp4 will be ignored
|
|
615
|
-
check_file_suffix = len(filetypes) > 1
|
|
616
|
-
|
|
617
581
|
try:
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
)
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
582
|
+
image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
|
|
583
|
+
# find descs that match the image paths from the import paths
|
|
584
|
+
image_metadatas = [
|
|
585
|
+
metadata
|
|
586
|
+
for metadata in (metadatas or [])
|
|
587
|
+
if isinstance(metadata, types.ImageMetadata)
|
|
588
|
+
]
|
|
589
|
+
specified_image_metadatas = _find_metadata_with_filename_existed_in(
|
|
590
|
+
image_metadatas, image_paths
|
|
591
|
+
)
|
|
592
|
+
if specified_image_metadatas:
|
|
593
|
+
try:
|
|
594
|
+
clusters = mly_uploader.upload_images(
|
|
595
|
+
specified_image_metadatas,
|
|
596
|
+
event_payload={"file_type": FileType.IMAGE.value},
|
|
597
|
+
)
|
|
598
|
+
except Exception as ex:
|
|
599
|
+
raise UploadError(ex) from ex
|
|
600
|
+
|
|
601
|
+
if clusters:
|
|
602
|
+
LOG.debug("Uploaded to cluster: %s", clusters)
|
|
603
|
+
|
|
604
|
+
video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
|
|
605
|
+
video_metadatas = [
|
|
606
|
+
metadata
|
|
607
|
+
for metadata in (metadatas or [])
|
|
608
|
+
if isinstance(metadata, types.VideoMetadata)
|
|
609
|
+
]
|
|
610
|
+
specified_video_metadatas = _find_metadata_with_filename_existed_in(
|
|
611
|
+
video_metadatas, video_paths
|
|
612
|
+
)
|
|
613
|
+
for idx, video_metadata in enumerate(specified_video_metadatas):
|
|
614
|
+
video_metadata.update_md5sum()
|
|
615
|
+
assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
|
|
616
|
+
|
|
617
|
+
# extract telemetry measurements from GoPro videos
|
|
618
|
+
telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = []
|
|
619
|
+
if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
|
|
620
|
+
if video_metadata.filetype is FileType.GOPRO:
|
|
621
|
+
with video_metadata.filename.open("rb") as fp:
|
|
622
|
+
telemetry_data = gpmf_parser.extract_telemetry_data(fp)
|
|
623
|
+
if telemetry_data:
|
|
624
|
+
telemetry_measurements.extend(telemetry_data.accl)
|
|
625
|
+
telemetry_measurements.extend(telemetry_data.gyro)
|
|
626
|
+
telemetry_measurements.extend(telemetry_data.magn)
|
|
627
|
+
telemetry_measurements.sort(key=lambda m: m.time)
|
|
628
|
+
|
|
629
|
+
generator = camm_builder.camm_sample_generator2(
|
|
630
|
+
video_metadata, telemetry_measurements=telemetry_measurements
|
|
632
631
|
)
|
|
633
|
-
|
|
632
|
+
|
|
633
|
+
with video_metadata.filename.open("rb") as src_fp:
|
|
634
|
+
camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
|
|
635
|
+
event_payload: uploader.Progress = {
|
|
636
|
+
"total_sequence_count": len(specified_video_metadatas),
|
|
637
|
+
"sequence_idx": idx,
|
|
638
|
+
"file_type": video_metadata.filetype.value,
|
|
639
|
+
"import_path": str(video_metadata.filename),
|
|
640
|
+
}
|
|
634
641
|
try:
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
642
|
+
cluster_id = mly_uploader.upload_stream(
|
|
643
|
+
T.cast(T.BinaryIO, camm_fp),
|
|
644
|
+
upload_api_v4.ClusterFileType.CAMM,
|
|
645
|
+
video_metadata.md5sum,
|
|
646
|
+
event_payload=event_payload,
|
|
638
647
|
)
|
|
639
648
|
except Exception as ex:
|
|
640
649
|
raise UploadError(ex) from ex
|
|
650
|
+
LOG.debug("Uploaded to cluster: %s", cluster_id)
|
|
641
651
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
supported = CAMM_CONVERTABLES.intersection(filetypes)
|
|
646
|
-
if supported:
|
|
647
|
-
video_paths = utils.find_videos(
|
|
648
|
-
import_paths,
|
|
649
|
-
skip_subfolders=skip_subfolders,
|
|
650
|
-
check_file_suffix=check_file_suffix,
|
|
651
|
-
)
|
|
652
|
-
video_metadatas = [
|
|
653
|
-
metadata
|
|
654
|
-
for metadata in (metadatas or [])
|
|
655
|
-
if isinstance(metadata, types.VideoMetadata)
|
|
656
|
-
]
|
|
657
|
-
specified_video_metadatas = _find_metadata_with_filename_existed_in(
|
|
658
|
-
video_metadatas, video_paths
|
|
659
|
-
)
|
|
660
|
-
for idx, video_metadata in enumerate(specified_video_metadatas):
|
|
661
|
-
video_metadata.update_md5sum()
|
|
662
|
-
assert isinstance(
|
|
663
|
-
video_metadata.md5sum, str
|
|
664
|
-
), "md5sum should be updated"
|
|
665
|
-
generator = camm_builder.camm_sample_generator2(video_metadata)
|
|
666
|
-
with video_metadata.filename.open("rb") as src_fp:
|
|
667
|
-
camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
|
|
668
|
-
event_payload: uploader.Progress = {
|
|
669
|
-
"total_sequence_count": len(specified_video_metadatas),
|
|
670
|
-
"sequence_idx": idx,
|
|
671
|
-
"file_type": video_metadata.filetype.value,
|
|
672
|
-
"import_path": str(video_metadata.filename),
|
|
673
|
-
}
|
|
674
|
-
try:
|
|
675
|
-
cluster_id = mly_uploader.upload_stream(
|
|
676
|
-
T.cast(T.BinaryIO, camm_fp),
|
|
677
|
-
upload_api_v4.ClusterFileType.CAMM,
|
|
678
|
-
video_metadata.md5sum,
|
|
679
|
-
event_payload=event_payload,
|
|
680
|
-
)
|
|
681
|
-
except Exception as ex:
|
|
682
|
-
raise UploadError(ex) from ex
|
|
683
|
-
LOG.debug("Uploaded to cluster: %s", cluster_id)
|
|
684
|
-
|
|
685
|
-
if DirectUploadFileType.RAW_BLACKVUE in filetypes:
|
|
686
|
-
video_paths = utils.find_videos(
|
|
687
|
-
import_paths,
|
|
688
|
-
skip_subfolders=skip_subfolders,
|
|
689
|
-
check_file_suffix=check_file_suffix,
|
|
690
|
-
)
|
|
691
|
-
_upload_raw_blackvues_DEPRECATED(mly_uploader, video_paths)
|
|
692
|
-
|
|
693
|
-
if DirectUploadFileType.RAW_CAMM in filetypes:
|
|
694
|
-
video_paths = utils.find_videos(
|
|
695
|
-
import_paths,
|
|
696
|
-
skip_subfolders=skip_subfolders,
|
|
697
|
-
check_file_suffix=check_file_suffix,
|
|
698
|
-
)
|
|
699
|
-
_upload_raw_camm_DEPRECATED(mly_uploader, video_paths)
|
|
700
|
-
|
|
701
|
-
if DirectUploadFileType.ZIP in filetypes:
|
|
702
|
-
zip_paths = utils.find_zipfiles(
|
|
703
|
-
import_paths,
|
|
704
|
-
skip_subfolders=skip_subfolders,
|
|
705
|
-
check_file_suffix=check_file_suffix,
|
|
706
|
-
)
|
|
707
|
-
_upload_zipfiles(mly_uploader, zip_paths)
|
|
652
|
+
zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
|
|
653
|
+
_upload_zipfiles(mly_uploader, zip_paths)
|
|
708
654
|
|
|
709
655
|
except UploadError as ex:
|
|
710
656
|
inner_ex = ex.inner_ex
|
|
@@ -743,111 +689,6 @@ def upload(
|
|
|
743
689
|
LOG.info("Nothing uploaded. Bye.")
|
|
744
690
|
|
|
745
691
|
|
|
746
|
-
def _check_blackvue_DEPRECATED(video_path: Path) -> None:
|
|
747
|
-
# Skip in tests only because we don't have valid sample blackvue for tests
|
|
748
|
-
if os.getenv("MAPILLARY__DISABLE_BLACKVUE_CHECK") == "YES":
|
|
749
|
-
return
|
|
750
|
-
|
|
751
|
-
points = blackvue_parser.parse_gps_points(video_path)
|
|
752
|
-
if not points:
|
|
753
|
-
raise exceptions.MapillaryGPXEmptyError("No GPS found in the BlackVue video")
|
|
754
|
-
|
|
755
|
-
stationary = video_utils.is_video_stationary(
|
|
756
|
-
geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
|
|
757
|
-
)
|
|
758
|
-
if stationary:
|
|
759
|
-
raise exceptions.MapillaryStationaryVideoError("Stationary BlackVue video")
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
def _upload_raw_blackvues_DEPRECATED(
|
|
763
|
-
mly_uploader: uploader.Uploader,
|
|
764
|
-
video_paths: T.Sequence[Path],
|
|
765
|
-
) -> None:
|
|
766
|
-
for idx, video_path in enumerate(video_paths):
|
|
767
|
-
event_payload: uploader.Progress = {
|
|
768
|
-
"total_sequence_count": len(video_paths),
|
|
769
|
-
"sequence_idx": idx,
|
|
770
|
-
"file_type": DirectUploadFileType.RAW_BLACKVUE.value,
|
|
771
|
-
"import_path": str(video_path),
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
try:
|
|
775
|
-
_check_blackvue_DEPRECATED(video_path)
|
|
776
|
-
except Exception as ex:
|
|
777
|
-
LOG.warning(
|
|
778
|
-
"Skipping %s %s due to: %s",
|
|
779
|
-
DirectUploadFileType.RAW_BLACKVUE.value.upper(),
|
|
780
|
-
video_path.name,
|
|
781
|
-
ex,
|
|
782
|
-
)
|
|
783
|
-
continue
|
|
784
|
-
|
|
785
|
-
with video_path.open("rb") as fp:
|
|
786
|
-
upload_md5sum = utils.md5sum_fp(fp).hexdigest()
|
|
787
|
-
try:
|
|
788
|
-
cluster_id = mly_uploader.upload_stream(
|
|
789
|
-
fp,
|
|
790
|
-
upload_api_v4.ClusterFileType.BLACKVUE,
|
|
791
|
-
upload_md5sum,
|
|
792
|
-
event_payload=event_payload,
|
|
793
|
-
)
|
|
794
|
-
except Exception as ex:
|
|
795
|
-
raise UploadError(ex) from ex
|
|
796
|
-
LOG.debug("Uploaded to cluster: %s", cluster_id)
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
def _check_camm_DEPRECATED(video_path: Path) -> None:
|
|
800
|
-
# Skip in tests only because we don't have valid sample CAMM for tests
|
|
801
|
-
if os.getenv("MAPILLARY__DISABLE_CAMM_CHECK") == "YES":
|
|
802
|
-
return
|
|
803
|
-
|
|
804
|
-
points = camm_parser.parse_gpx(video_path)
|
|
805
|
-
if not points:
|
|
806
|
-
raise exceptions.MapillaryGPXEmptyError("No GPS found in the CAMM video")
|
|
807
|
-
|
|
808
|
-
stationary = video_utils.is_video_stationary(
|
|
809
|
-
geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
|
|
810
|
-
)
|
|
811
|
-
if stationary:
|
|
812
|
-
raise exceptions.MapillaryStationaryVideoError("Stationary CAMM video")
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
def _upload_raw_camm_DEPRECATED(
|
|
816
|
-
mly_uploader: uploader.Uploader,
|
|
817
|
-
video_paths: T.Sequence[Path],
|
|
818
|
-
) -> None:
|
|
819
|
-
for idx, video_path in enumerate(video_paths):
|
|
820
|
-
event_payload: uploader.Progress = {
|
|
821
|
-
"total_sequence_count": len(video_paths),
|
|
822
|
-
"sequence_idx": idx,
|
|
823
|
-
"file_type": DirectUploadFileType.RAW_CAMM.value,
|
|
824
|
-
"import_path": str(video_path),
|
|
825
|
-
}
|
|
826
|
-
try:
|
|
827
|
-
_check_camm_DEPRECATED(video_path)
|
|
828
|
-
except Exception as ex:
|
|
829
|
-
LOG.warning(
|
|
830
|
-
"Skipping %s %s due to: %s",
|
|
831
|
-
DirectUploadFileType.RAW_CAMM.value.upper(),
|
|
832
|
-
video_path.name,
|
|
833
|
-
ex,
|
|
834
|
-
)
|
|
835
|
-
continue
|
|
836
|
-
try:
|
|
837
|
-
with video_path.open("rb") as fp:
|
|
838
|
-
upload_md5sum = utils.md5sum_fp(fp).hexdigest()
|
|
839
|
-
cluster_id = mly_uploader.upload_stream(
|
|
840
|
-
fp,
|
|
841
|
-
upload_api_v4.ClusterFileType.CAMM,
|
|
842
|
-
upload_md5sum,
|
|
843
|
-
event_payload=event_payload,
|
|
844
|
-
)
|
|
845
|
-
except Exception as ex:
|
|
846
|
-
raise UploadError(ex) from ex
|
|
847
|
-
|
|
848
|
-
LOG.debug("Uploaded to cluster: %s", cluster_id)
|
|
849
|
-
|
|
850
|
-
|
|
851
692
|
def _upload_zipfiles(
|
|
852
693
|
mly_uploader: uploader.Uploader,
|
|
853
694
|
zip_paths: T.Sequence[Path],
|
|
@@ -856,7 +697,7 @@ def _upload_zipfiles(
|
|
|
856
697
|
event_payload: uploader.Progress = {
|
|
857
698
|
"total_sequence_count": len(zip_paths),
|
|
858
699
|
"sequence_idx": idx,
|
|
859
|
-
"file_type":
|
|
700
|
+
"file_type": FileType.ZIP.value,
|
|
860
701
|
"import_path": str(zip_path),
|
|
861
702
|
}
|
|
862
703
|
try:
|