mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapillary_tools/__init__.py +1 -1
- mapillary_tools/api_v4.py +94 -4
- mapillary_tools/{geotag → camm}/camm_builder.py +122 -62
- mapillary_tools/{geotag → camm}/camm_parser.py +120 -84
- mapillary_tools/commands/__init__.py +0 -1
- mapillary_tools/commands/__main__.py +0 -6
- mapillary_tools/commands/process.py +0 -50
- mapillary_tools/commands/upload.py +1 -26
- mapillary_tools/constants.py +2 -2
- mapillary_tools/exiftool_read_video.py +13 -11
- mapillary_tools/ffmpeg.py +2 -2
- mapillary_tools/geo.py +0 -54
- mapillary_tools/geotag/blackvue_parser.py +4 -4
- mapillary_tools/geotag/geotag_images_from_exif.py +2 -1
- mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -1
- mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +5 -3
- mapillary_tools/geotag/geotag_videos_from_video.py +13 -14
- mapillary_tools/geotag/gpmf_gps_filter.py +9 -10
- mapillary_tools/geotag/gpmf_parser.py +346 -83
- mapillary_tools/mp4/__init__.py +0 -0
- mapillary_tools/{geotag → mp4}/construct_mp4_parser.py +32 -16
- mapillary_tools/mp4/mp4_sample_parser.py +322 -0
- mapillary_tools/{geotag → mp4}/simple_mp4_builder.py +64 -38
- mapillary_tools/process_geotag_properties.py +25 -19
- mapillary_tools/process_sequence_properties.py +6 -6
- mapillary_tools/sample_video.py +17 -16
- mapillary_tools/telemetry.py +59 -0
- mapillary_tools/types.py +18 -0
- mapillary_tools/upload.py +75 -233
- mapillary_tools/upload_api_v4.py +8 -9
- mapillary_tools/utils.py +9 -16
- mapillary_tools/video_data_extraction/cli_options.py +0 -1
- mapillary_tools/video_data_extraction/extract_video_data.py +13 -31
- mapillary_tools/video_data_extraction/extractors/base_parser.py +13 -11
- mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +5 -4
- mapillary_tools/video_data_extraction/extractors/camm_parser.py +7 -6
- mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +4 -9
- mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +9 -11
- mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +6 -11
- mapillary_tools/video_data_extraction/extractors/gopro_parser.py +11 -4
- mapillary_tools/video_data_extraction/extractors/gpx_parser.py +54 -12
- mapillary_tools/video_data_extraction/extractors/nmea_parser.py +3 -3
- mapillary_tools/video_data_extraction/video_data_parser_factory.py +13 -20
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/METADATA +10 -3
- mapillary_tools-0.13.1a1.dist-info/RECORD +75 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/WHEEL +1 -1
- mapillary_tools/commands/upload_blackvue.py +0 -33
- mapillary_tools/commands/upload_camm.py +0 -33
- mapillary_tools/commands/upload_zip.py +0 -33
- mapillary_tools/geotag/mp4_sample_parser.py +0 -426
- mapillary_tools/process_import_meta_properties.py +0 -76
- mapillary_tools-0.12.1.dist-info/RECORD +0 -77
- /mapillary_tools/{geotag → mp4}/io_utils.py +0 -0
- /mapillary_tools/{geotag → mp4}/simple_mp4_parser.py +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/LICENSE +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/entry_points.txt +0 -0
- {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/top_level.txt +0 -0
|
@@ -9,17 +9,22 @@ from enum import Enum
|
|
|
9
9
|
|
|
10
10
|
import construct as C
|
|
11
11
|
|
|
12
|
-
from
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
mp4_sample_parser as sample_parser,
|
|
16
|
-
simple_mp4_parser as parser,
|
|
17
|
-
)
|
|
12
|
+
from .. import geo, telemetry
|
|
13
|
+
from ..mp4 import simple_mp4_parser as sparser
|
|
14
|
+
from ..mp4.mp4_sample_parser import MovieBoxParser, Sample, TrackBoxParser
|
|
18
15
|
|
|
19
16
|
|
|
20
17
|
LOG = logging.getLogger(__name__)
|
|
21
18
|
|
|
22
19
|
|
|
20
|
+
TelemetryMeasurement = T.Union[
|
|
21
|
+
geo.Point,
|
|
22
|
+
telemetry.AccelerationData,
|
|
23
|
+
telemetry.GyroscopeData,
|
|
24
|
+
telemetry.MagnetometerData,
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
23
28
|
# Camera Motion Metadata Spec https://developers.google.com/streetview/publish/camm-spec
|
|
24
29
|
class CAMMType(Enum):
|
|
25
30
|
ANGLE_AXIS = 0
|
|
@@ -79,15 +84,15 @@ CAMMSampleData = C.Struct(
|
|
|
79
84
|
)
|
|
80
85
|
|
|
81
86
|
|
|
82
|
-
def
|
|
83
|
-
fp: T.BinaryIO, sample:
|
|
84
|
-
) -> T.Optional[
|
|
85
|
-
fp.seek(sample.offset, io.SEEK_SET)
|
|
86
|
-
data = fp.read(sample.size)
|
|
87
|
+
def _parse_telemetry_from_sample(
|
|
88
|
+
fp: T.BinaryIO, sample: Sample
|
|
89
|
+
) -> T.Optional[TelemetryMeasurement]:
|
|
90
|
+
fp.seek(sample.raw_sample.offset, io.SEEK_SET)
|
|
91
|
+
data = fp.read(sample.raw_sample.size)
|
|
87
92
|
box = CAMMSampleData.parse(data)
|
|
88
93
|
if box.type == CAMMType.MIN_GPS.value:
|
|
89
94
|
return geo.Point(
|
|
90
|
-
time=sample.
|
|
95
|
+
time=sample.exact_time,
|
|
91
96
|
lat=box.data[0],
|
|
92
97
|
lon=box.data[1],
|
|
93
98
|
alt=box.data[2],
|
|
@@ -97,18 +102,40 @@ def _parse_point_from_sample(
|
|
|
97
102
|
# Not using box.data.time_gps_epoch as the point timestamp
|
|
98
103
|
# because it is from another clock
|
|
99
104
|
return geo.Point(
|
|
100
|
-
time=sample.
|
|
105
|
+
time=sample.exact_time,
|
|
101
106
|
lat=box.data.latitude,
|
|
102
107
|
lon=box.data.longitude,
|
|
103
108
|
alt=box.data.altitude,
|
|
104
109
|
angle=None,
|
|
105
110
|
)
|
|
111
|
+
elif box.type == CAMMType.ACCELERATION.value:
|
|
112
|
+
return telemetry.AccelerationData(
|
|
113
|
+
time=sample.exact_time,
|
|
114
|
+
x=box.data[0],
|
|
115
|
+
y=box.data[1],
|
|
116
|
+
z=box.data[2],
|
|
117
|
+
)
|
|
118
|
+
elif box.type == CAMMType.GYRO.value:
|
|
119
|
+
return telemetry.GyroscopeData(
|
|
120
|
+
time=sample.exact_time,
|
|
121
|
+
x=box.data[0],
|
|
122
|
+
y=box.data[1],
|
|
123
|
+
z=box.data[2],
|
|
124
|
+
)
|
|
125
|
+
elif box.type == CAMMType.MAGNETIC_FIELD.value:
|
|
126
|
+
return telemetry.MagnetometerData(
|
|
127
|
+
time=sample.exact_time,
|
|
128
|
+
x=box.data[0],
|
|
129
|
+
y=box.data[1],
|
|
130
|
+
z=box.data[2],
|
|
131
|
+
)
|
|
106
132
|
return None
|
|
107
133
|
|
|
108
134
|
|
|
109
|
-
def
|
|
110
|
-
|
|
111
|
-
|
|
135
|
+
def _filter_telemetry_by_elst_segments(
|
|
136
|
+
measurements: T.Iterable[TelemetryMeasurement],
|
|
137
|
+
elst: T.Sequence[T.Tuple[float, float]],
|
|
138
|
+
) -> T.Generator[TelemetryMeasurement, None, None]:
|
|
112
139
|
empty_elst = [entry for entry in elst if entry[0] == -1]
|
|
113
140
|
if empty_elst:
|
|
114
141
|
offset = empty_elst[-1][1]
|
|
@@ -118,20 +145,20 @@ def filter_points_by_elst(
|
|
|
118
145
|
elst = [entry for entry in elst if entry[0] != -1]
|
|
119
146
|
|
|
120
147
|
if not elst:
|
|
121
|
-
for
|
|
122
|
-
yield dataclasses.replace(
|
|
148
|
+
for m in measurements:
|
|
149
|
+
yield dataclasses.replace(m, time=m.time + offset)
|
|
123
150
|
return
|
|
124
151
|
|
|
125
152
|
elst.sort(key=lambda entry: entry[0])
|
|
126
153
|
elst_idx = 0
|
|
127
|
-
for
|
|
154
|
+
for m in measurements:
|
|
128
155
|
if len(elst) <= elst_idx:
|
|
129
156
|
break
|
|
130
157
|
media_time, duration = elst[elst_idx]
|
|
131
|
-
if
|
|
158
|
+
if m.time < media_time:
|
|
132
159
|
pass
|
|
133
|
-
elif
|
|
134
|
-
yield dataclasses.replace(
|
|
160
|
+
elif m.time <= media_time + duration:
|
|
161
|
+
yield dataclasses.replace(m, time=m.time + offset)
|
|
135
162
|
else:
|
|
136
163
|
elst_idx += 1
|
|
137
164
|
|
|
@@ -148,15 +175,45 @@ def elst_entry_to_seconds(
|
|
|
148
175
|
return (media_time, duration)
|
|
149
176
|
|
|
150
177
|
|
|
151
|
-
def
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
178
|
+
def _is_camm_description(description: T.Dict) -> bool:
|
|
179
|
+
return description["format"] == b"camm"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _contains_camm_description(track: TrackBoxParser) -> bool:
|
|
183
|
+
descriptions = track.extract_sample_descriptions()
|
|
184
|
+
return any(_is_camm_description(d) for d in descriptions)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _filter_telemetry_by_track_elst(
|
|
188
|
+
moov: MovieBoxParser,
|
|
189
|
+
track: TrackBoxParser,
|
|
190
|
+
measurements: T.Iterable[TelemetryMeasurement],
|
|
191
|
+
) -> T.List[TelemetryMeasurement]:
|
|
192
|
+
elst_boxdata = track.extract_elst_boxdata()
|
|
193
|
+
|
|
194
|
+
if elst_boxdata is not None:
|
|
195
|
+
elst_entries = elst_boxdata["entries"]
|
|
196
|
+
if elst_entries:
|
|
197
|
+
# media_timescale
|
|
198
|
+
mdhd_boxdata = track.extract_mdhd_boxdata()
|
|
199
|
+
media_timescale = mdhd_boxdata["timescale"]
|
|
200
|
+
|
|
201
|
+
# movie_timescale
|
|
202
|
+
mvhd_boxdata = moov.extract_mvhd_boxdata()
|
|
203
|
+
movie_timescale = mvhd_boxdata["timescale"]
|
|
204
|
+
|
|
205
|
+
segments = [
|
|
206
|
+
elst_entry_to_seconds(
|
|
207
|
+
entry,
|
|
208
|
+
movie_timescale=movie_timescale,
|
|
209
|
+
media_timescale=media_timescale,
|
|
210
|
+
)
|
|
211
|
+
for entry in elst_entries
|
|
212
|
+
]
|
|
213
|
+
|
|
214
|
+
return list(_filter_telemetry_by_elst_segments(measurements, segments))
|
|
215
|
+
|
|
216
|
+
return list(measurements)
|
|
160
217
|
|
|
161
218
|
|
|
162
219
|
def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
|
|
@@ -165,62 +222,41 @@ def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
|
|
|
165
222
|
otherwise None
|
|
166
223
|
"""
|
|
167
224
|
|
|
168
|
-
|
|
169
|
-
movie_timescale = None
|
|
170
|
-
media_timescale = None
|
|
171
|
-
elst_entries = None
|
|
225
|
+
moov = MovieBoxParser.parse_stream(fp)
|
|
172
226
|
|
|
173
|
-
for
|
|
174
|
-
if
|
|
175
|
-
|
|
227
|
+
for track in moov.extract_tracks():
|
|
228
|
+
if _contains_camm_description(track):
|
|
229
|
+
maybe_measurements = (
|
|
230
|
+
_parse_telemetry_from_sample(fp, sample)
|
|
231
|
+
for sample in track.extract_samples()
|
|
232
|
+
if _is_camm_description(sample.description)
|
|
233
|
+
)
|
|
234
|
+
points = [m for m in maybe_measurements if isinstance(m, geo.Point)]
|
|
176
235
|
|
|
177
|
-
|
|
178
|
-
|
|
236
|
+
return T.cast(
|
|
237
|
+
T.List[geo.Point], _filter_telemetry_by_track_elst(moov, track, points)
|
|
179
238
|
)
|
|
180
|
-
camm_descriptions = [d for d in descriptions if d["format"] == b"camm"]
|
|
181
|
-
if camm_descriptions:
|
|
182
|
-
s.seek(trak_start_offset, io.SEEK_SET)
|
|
183
|
-
camm_samples = _extract_camm_samples(s, h.maxsize)
|
|
184
|
-
|
|
185
|
-
points_with_nones = (
|
|
186
|
-
_parse_point_from_sample(fp, sample)
|
|
187
|
-
for sample in camm_samples
|
|
188
|
-
if sample.description["format"] == b"camm"
|
|
189
|
-
)
|
|
190
239
|
|
|
191
|
-
|
|
192
|
-
if points:
|
|
193
|
-
s.seek(trak_start_offset)
|
|
194
|
-
elst_data = parser.parse_box_data_first(
|
|
195
|
-
s, [b"edts", b"elst"], maxsize=h.maxsize
|
|
196
|
-
)
|
|
197
|
-
if elst_data is not None:
|
|
198
|
-
elst_entries = cparser.EditBox.parse(elst_data)["entries"]
|
|
199
|
-
|
|
200
|
-
s.seek(trak_start_offset)
|
|
201
|
-
mdhd_data = parser.parse_box_data_firstx(
|
|
202
|
-
s, [b"mdia", b"mdhd"], maxsize=h.maxsize
|
|
203
|
-
)
|
|
204
|
-
mdhd = cparser.MediaHeaderBox.parse(mdhd_data)
|
|
205
|
-
media_timescale = mdhd["timescale"]
|
|
206
|
-
else:
|
|
207
|
-
assert h.type == b"mvhd"
|
|
208
|
-
if not movie_timescale:
|
|
209
|
-
mvhd = cparser.MovieHeaderBox.parse(s.read(h.maxsize))
|
|
210
|
-
movie_timescale = mvhd["timescale"]
|
|
240
|
+
return None
|
|
211
241
|
|
|
212
|
-
# exit when both found
|
|
213
|
-
if movie_timescale is not None and points:
|
|
214
|
-
break
|
|
215
242
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
elst_entry_to_seconds(entry, movie_timescale, media_timescale)
|
|
219
|
-
for entry in elst_entries
|
|
220
|
-
]
|
|
221
|
-
points = list(filter_points_by_elst(points, segments))
|
|
243
|
+
def extract_telemetry_data(fp: T.BinaryIO) -> T.Optional[T.List[TelemetryMeasurement]]:
|
|
244
|
+
moov = MovieBoxParser.parse_stream(fp)
|
|
222
245
|
|
|
223
|
-
|
|
246
|
+
for track in moov.extract_tracks():
|
|
247
|
+
if _contains_camm_description(track):
|
|
248
|
+
maybe_measurements = (
|
|
249
|
+
_parse_telemetry_from_sample(fp, sample)
|
|
250
|
+
for sample in track.extract_samples()
|
|
251
|
+
if _is_camm_description(sample.description)
|
|
252
|
+
)
|
|
253
|
+
measurements = [m for m in maybe_measurements if m is not None]
|
|
254
|
+
|
|
255
|
+
measurements = _filter_telemetry_by_track_elst(moov, track, measurements)
|
|
256
|
+
|
|
257
|
+
return measurements
|
|
258
|
+
|
|
259
|
+
return None
|
|
224
260
|
|
|
225
261
|
|
|
226
262
|
def parse_gpx(path: pathlib.Path) -> T.List[geo.Point]:
|
|
@@ -238,7 +274,7 @@ MakeOrModel = C.Struct(
|
|
|
238
274
|
)
|
|
239
275
|
|
|
240
276
|
|
|
241
|
-
def _decode_quietly(data: bytes, h:
|
|
277
|
+
def _decode_quietly(data: bytes, h: sparser.Header) -> str:
|
|
242
278
|
try:
|
|
243
279
|
return data.decode("utf-8")
|
|
244
280
|
except UnicodeDecodeError:
|
|
@@ -246,7 +282,7 @@ def _decode_quietly(data: bytes, h: parser.Header) -> str:
|
|
|
246
282
|
return ""
|
|
247
283
|
|
|
248
284
|
|
|
249
|
-
def _parse_quietly(data: bytes, h:
|
|
285
|
+
def _parse_quietly(data: bytes, h: sparser.Header) -> bytes:
|
|
250
286
|
try:
|
|
251
287
|
parsed = MakeOrModel.parse(data)
|
|
252
288
|
except C.ConstructError:
|
|
@@ -256,7 +292,7 @@ def _parse_quietly(data: bytes, h: parser.Header) -> bytes:
|
|
|
256
292
|
|
|
257
293
|
|
|
258
294
|
def extract_camera_make_and_model(fp: T.BinaryIO) -> T.Tuple[str, str]:
|
|
259
|
-
header_and_stream =
|
|
295
|
+
header_and_stream = sparser.parse_path(
|
|
260
296
|
fp,
|
|
261
297
|
[
|
|
262
298
|
b"moov",
|
|
@@ -296,7 +332,7 @@ def extract_camera_make_and_model(fp: T.BinaryIO) -> T.Tuple[str, str]:
|
|
|
296
332
|
# quit when both found
|
|
297
333
|
if make and model:
|
|
298
334
|
break
|
|
299
|
-
except
|
|
335
|
+
except sparser.ParsingError:
|
|
300
336
|
pass
|
|
301
337
|
|
|
302
338
|
if make:
|
|
@@ -12,9 +12,6 @@ from . import (
|
|
|
12
12
|
process_and_upload,
|
|
13
13
|
sample_video,
|
|
14
14
|
upload,
|
|
15
|
-
upload_blackvue,
|
|
16
|
-
upload_camm,
|
|
17
|
-
upload_zip,
|
|
18
15
|
video_process,
|
|
19
16
|
video_process_and_upload,
|
|
20
17
|
zip,
|
|
@@ -23,9 +20,6 @@ from . import (
|
|
|
23
20
|
mapillary_tools_commands = [
|
|
24
21
|
process,
|
|
25
22
|
upload,
|
|
26
|
-
upload_camm,
|
|
27
|
-
upload_blackvue,
|
|
28
|
-
upload_zip,
|
|
29
23
|
sample_video,
|
|
30
24
|
video_process,
|
|
31
25
|
authenticate,
|
|
@@ -10,7 +10,6 @@ from ..process_geotag_properties import (
|
|
|
10
10
|
process_finalize,
|
|
11
11
|
process_geotag_properties,
|
|
12
12
|
)
|
|
13
|
-
from ..process_import_meta_properties import process_import_meta_properties
|
|
14
13
|
from ..process_sequence_properties import process_sequence_properties
|
|
15
14
|
|
|
16
15
|
|
|
@@ -107,44 +106,6 @@ class Command:
|
|
|
107
106
|
default=None,
|
|
108
107
|
required=False,
|
|
109
108
|
)
|
|
110
|
-
group_metadata.add_argument(
|
|
111
|
-
"--add_file_name",
|
|
112
|
-
help="[DEPRECATED since v0.9.4] Add original file name to EXIF.",
|
|
113
|
-
action="store_true",
|
|
114
|
-
required=False,
|
|
115
|
-
)
|
|
116
|
-
group_metadata.add_argument(
|
|
117
|
-
"--add_import_date",
|
|
118
|
-
help="[DEPRECATED since v0.10.0] Add import date.",
|
|
119
|
-
action="store_true",
|
|
120
|
-
required=False,
|
|
121
|
-
)
|
|
122
|
-
group_metadata.add_argument(
|
|
123
|
-
"--orientation",
|
|
124
|
-
help="Specify the image orientation in degrees. Note this might result in image rotation. Note this input has precedence over the input read from the import source file.",
|
|
125
|
-
choices=[0, 90, 180, 270],
|
|
126
|
-
type=int,
|
|
127
|
-
default=None,
|
|
128
|
-
required=False,
|
|
129
|
-
)
|
|
130
|
-
group_metadata.add_argument(
|
|
131
|
-
"--GPS_accuracy",
|
|
132
|
-
help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.",
|
|
133
|
-
default=None,
|
|
134
|
-
required=False,
|
|
135
|
-
)
|
|
136
|
-
group_metadata.add_argument(
|
|
137
|
-
"--camera_uuid",
|
|
138
|
-
help="Custom string used to differentiate different captures taken with the same camera make and model.",
|
|
139
|
-
default=None,
|
|
140
|
-
required=False,
|
|
141
|
-
)
|
|
142
|
-
group_metadata.add_argument(
|
|
143
|
-
"--custom_meta_data",
|
|
144
|
-
help='[DEPRECATED since v0.10.0] Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234".',
|
|
145
|
-
default=None,
|
|
146
|
-
required=False,
|
|
147
|
-
)
|
|
148
109
|
|
|
149
110
|
group_geotagging = parser.add_argument_group(
|
|
150
111
|
f"{constants.ANSI_BOLD}PROCESS GEOTAGGING OPTIONS{constants.ANSI_RESET_ALL}"
|
|
@@ -278,17 +239,6 @@ class Command:
|
|
|
278
239
|
),
|
|
279
240
|
)
|
|
280
241
|
|
|
281
|
-
metadatas = process_import_meta_properties(
|
|
282
|
-
metadatas=metadatas,
|
|
283
|
-
**(
|
|
284
|
-
{
|
|
285
|
-
k: v
|
|
286
|
-
for k, v in vars_args.items()
|
|
287
|
-
if k in inspect.getfullargspec(process_import_meta_properties).args
|
|
288
|
-
}
|
|
289
|
-
),
|
|
290
|
-
)
|
|
291
|
-
|
|
292
242
|
metadatas = process_sequence_properties(
|
|
293
243
|
metadatas=metadatas,
|
|
294
244
|
**(
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
|
-
import typing as T
|
|
3
2
|
|
|
4
3
|
from .. import constants
|
|
5
|
-
from ..upload import
|
|
4
|
+
from ..upload import upload
|
|
6
5
|
|
|
7
6
|
|
|
8
7
|
class Command:
|
|
@@ -34,30 +33,6 @@ class Command:
|
|
|
34
33
|
group = parser.add_argument_group(
|
|
35
34
|
f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}"
|
|
36
35
|
)
|
|
37
|
-
default_filetypes = ",".join(sorted(t.value for t in FileType))
|
|
38
|
-
supported_filetypes = ",".join(
|
|
39
|
-
sorted(
|
|
40
|
-
[t.value for t in DirectUploadFileType] + [t.value for t in FileType]
|
|
41
|
-
)
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
def _type(option: str) -> T.List[T.Union[FileType, DirectUploadFileType]]:
|
|
45
|
-
r: T.List[T.Union[FileType, DirectUploadFileType]] = []
|
|
46
|
-
for t in option.split(","):
|
|
47
|
-
if t in [x.value for x in FileType]:
|
|
48
|
-
r.append(FileType(t))
|
|
49
|
-
else:
|
|
50
|
-
r.append(DirectUploadFileType(t))
|
|
51
|
-
return r
|
|
52
|
-
|
|
53
|
-
group.add_argument(
|
|
54
|
-
"--filetypes",
|
|
55
|
-
"--file_types",
|
|
56
|
-
help=f"Upload files of the specified types only. Supported file types: {supported_filetypes} [default: %(default)s]",
|
|
57
|
-
type=_type,
|
|
58
|
-
default=default_filetypes,
|
|
59
|
-
required=False,
|
|
60
|
-
)
|
|
61
36
|
group.add_argument(
|
|
62
37
|
"--desc_path",
|
|
63
38
|
help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
|
mapillary_tools/constants.py
CHANGED
|
@@ -43,8 +43,8 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
|
|
|
43
43
|
|
|
44
44
|
# WARNING: Changing the following envvars might result in failed uploads
|
|
45
45
|
# Max number of images per sequence
|
|
46
|
-
MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH",
|
|
46
|
+
MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
|
|
47
47
|
# Max file size per sequence (sum of image filesizes in the sequence)
|
|
48
|
-
MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "
|
|
48
|
+
MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
|
|
49
49
|
# Max number of pixels per sequence (sum of image pixels in the sequence)
|
|
50
50
|
MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
|
|
@@ -5,6 +5,7 @@ import typing as T
|
|
|
5
5
|
import xml.etree.ElementTree as ET
|
|
6
6
|
|
|
7
7
|
from . import exif_read, exiftool_read, geo
|
|
8
|
+
from .telemetry import GPSFix, GPSPoint
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
MAX_TRACK_ID = 10
|
|
@@ -87,7 +88,7 @@ def _aggregate_gps_track(
|
|
|
87
88
|
alt_tag: T.Optional[str] = None,
|
|
88
89
|
direction_tag: T.Optional[str] = None,
|
|
89
90
|
ground_speed_tag: T.Optional[str] = None,
|
|
90
|
-
) -> T.List[
|
|
91
|
+
) -> T.List[GPSPoint]:
|
|
91
92
|
"""
|
|
92
93
|
Aggregate all GPS data by the tags.
|
|
93
94
|
It requires lat, lon to be present, and their lengths must match.
|
|
@@ -173,15 +174,16 @@ def _aggregate_gps_track(
|
|
|
173
174
|
if timestamp is None or lon is None or lat is None:
|
|
174
175
|
continue
|
|
175
176
|
track.append(
|
|
176
|
-
|
|
177
|
+
GPSPoint(
|
|
177
178
|
time=timestamp,
|
|
178
179
|
lon=lon,
|
|
179
180
|
lat=lat,
|
|
180
181
|
alt=alt,
|
|
181
182
|
angle=direction,
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
183
|
+
epoch_time=None,
|
|
184
|
+
fix=None,
|
|
185
|
+
precision=None,
|
|
186
|
+
ground_speed=ground_speed,
|
|
185
187
|
)
|
|
186
188
|
)
|
|
187
189
|
|
|
@@ -230,8 +232,8 @@ def _aggregate_gps_track_by_sample_time(
|
|
|
230
232
|
ground_speed_tag: T.Optional[str] = None,
|
|
231
233
|
gps_fix_tag: T.Optional[str] = None,
|
|
232
234
|
gps_precision_tag: T.Optional[str] = None,
|
|
233
|
-
) -> T.List[
|
|
234
|
-
track: T.List[
|
|
235
|
+
) -> T.List[GPSPoint]:
|
|
236
|
+
track: T.List[GPSPoint] = []
|
|
235
237
|
|
|
236
238
|
expanded_gps_fix_tag = None
|
|
237
239
|
if gps_fix_tag is not None:
|
|
@@ -249,7 +251,7 @@ def _aggregate_gps_track_by_sample_time(
|
|
|
249
251
|
gps_fix_texts = texts_by_tag.get(expanded_gps_fix_tag)
|
|
250
252
|
if gps_fix_texts:
|
|
251
253
|
try:
|
|
252
|
-
gps_fix =
|
|
254
|
+
gps_fix = GPSFix(int(gps_fix_texts[0]))
|
|
253
255
|
except ValueError:
|
|
254
256
|
gps_fix = None
|
|
255
257
|
|
|
@@ -280,7 +282,7 @@ def _aggregate_gps_track_by_sample_time(
|
|
|
280
282
|
for idx, point in enumerate(points):
|
|
281
283
|
point.time = sample_time + idx * avg_timedelta
|
|
282
284
|
track.extend(
|
|
283
|
-
dataclasses.replace(point,
|
|
285
|
+
dataclasses.replace(point, fix=gps_fix, precision=gps_precision)
|
|
284
286
|
for point in points
|
|
285
287
|
)
|
|
286
288
|
|
|
@@ -355,7 +357,7 @@ class ExifToolReadVideo:
|
|
|
355
357
|
_, model = self._extract_make_and_model()
|
|
356
358
|
return model
|
|
357
359
|
|
|
358
|
-
def _extract_gps_track_from_track(self) -> T.List[
|
|
360
|
+
def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
|
|
359
361
|
for track_id in range(1, MAX_TRACK_ID + 1):
|
|
360
362
|
track_ns = f"Track{track_id}"
|
|
361
363
|
if self._all_tags_exists(
|
|
@@ -397,7 +399,7 @@ class ExifToolReadVideo:
|
|
|
397
399
|
|
|
398
400
|
def _extract_gps_track_from_quicktime(
|
|
399
401
|
self, namespace: str = "QuickTime"
|
|
400
|
-
) -> T.List[
|
|
402
|
+
) -> T.List[GPSPoint]:
|
|
401
403
|
if not self._all_tags_exists(
|
|
402
404
|
{
|
|
403
405
|
expand_tag(f"{namespace}:GPSDateTime"),
|
mapillary_tools/ffmpeg.py
CHANGED
|
@@ -202,10 +202,10 @@ class FFMPEG:
|
|
|
202
202
|
return "0"
|
|
203
203
|
|
|
204
204
|
if length == 1:
|
|
205
|
-
return f"eq(n\\,{
|
|
205
|
+
return f"eq(n\\,{sorted_frame_indices[0]})"
|
|
206
206
|
|
|
207
207
|
middle = length // 2
|
|
208
|
-
return f"if(lt(n\\,{
|
|
208
|
+
return f"if(lt(n\\,{sorted_frame_indices[middle]})\\,{self.generate_binary_search(sorted_frame_indices[:middle])}\\,{self.generate_binary_search(sorted_frame_indices[middle:])})"
|
|
209
209
|
|
|
210
210
|
def extract_specified_frames(
|
|
211
211
|
self,
|
mapillary_tools/geo.py
CHANGED
|
@@ -6,7 +6,6 @@ import datetime
|
|
|
6
6
|
import itertools
|
|
7
7
|
import math
|
|
8
8
|
import typing as T
|
|
9
|
-
from enum import Enum, unique
|
|
10
9
|
|
|
11
10
|
WGS84_a = 6378137.0
|
|
12
11
|
WGS84_a_SQ = WGS84_a**2
|
|
@@ -32,45 +31,6 @@ class Point:
|
|
|
32
31
|
angle: T.Optional[float]
|
|
33
32
|
|
|
34
33
|
|
|
35
|
-
@unique
|
|
36
|
-
class GPSFix(Enum):
|
|
37
|
-
NO_FIX = 0
|
|
38
|
-
FIX_2D = 2
|
|
39
|
-
FIX_3D = 3
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
@dataclasses.dataclass
|
|
43
|
-
class PointWithFix(Point):
|
|
44
|
-
gps_fix: T.Optional[GPSFix]
|
|
45
|
-
gps_precision: T.Optional[float]
|
|
46
|
-
gps_ground_speed: T.Optional[float]
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def _ecef_from_lla_DEPRECATED(
|
|
50
|
-
lat: float, lon: float, alt: float
|
|
51
|
-
) -> T.Tuple[float, float, float]:
|
|
52
|
-
"""
|
|
53
|
-
Deprecated because it is slow. Keep here for reference and comparison.
|
|
54
|
-
Use _ecef_from_lla2 instead.
|
|
55
|
-
|
|
56
|
-
Compute ECEF XYZ from latitude, longitude and altitude.
|
|
57
|
-
|
|
58
|
-
All using the WGS94 model.
|
|
59
|
-
Altitude is the distance to the WGS94 ellipsoid.
|
|
60
|
-
Check results here http://www.oc.nps.edu/oc2902w/coord/llhxyz.htm
|
|
61
|
-
|
|
62
|
-
"""
|
|
63
|
-
a2 = WGS84_a**2
|
|
64
|
-
b2 = WGS84_b**2
|
|
65
|
-
lat = math.radians(lat)
|
|
66
|
-
lon = math.radians(lon)
|
|
67
|
-
L = 1.0 / math.sqrt(a2 * math.cos(lat) ** 2 + b2 * math.sin(lat) ** 2)
|
|
68
|
-
x = (a2 * L + alt) * math.cos(lat) * math.cos(lon)
|
|
69
|
-
y = (a2 * L + alt) * math.cos(lat) * math.sin(lon)
|
|
70
|
-
z = (b2 * L + alt) * math.sin(lat)
|
|
71
|
-
return x, y, z
|
|
72
|
-
|
|
73
|
-
|
|
74
34
|
def _ecef_from_lla2(lat: float, lon: float) -> T.Tuple[float, float, float]:
|
|
75
35
|
"""
|
|
76
36
|
Compute ECEF XYZ from latitude, longitude and altitude.
|
|
@@ -172,20 +132,6 @@ def pairwise(iterable: T.Iterable[_IT]) -> T.Iterable[T.Tuple[_IT, _IT]]:
|
|
|
172
132
|
return zip(a, b)
|
|
173
133
|
|
|
174
134
|
|
|
175
|
-
def group_every(
|
|
176
|
-
iterable: T.Iterable[_IT], n: int
|
|
177
|
-
) -> T.Generator[T.Generator[_IT, None, None], None, None]:
|
|
178
|
-
"""
|
|
179
|
-
Return a generator that divides the iterable into groups by N.
|
|
180
|
-
"""
|
|
181
|
-
|
|
182
|
-
if not (0 < n):
|
|
183
|
-
raise ValueError("expect 0 < n but got {0}".format(n))
|
|
184
|
-
|
|
185
|
-
for _, group in itertools.groupby(enumerate(iterable), key=lambda t: t[0] // n):
|
|
186
|
-
yield (item for _, item in group)
|
|
187
|
-
|
|
188
|
-
|
|
189
135
|
def as_unix_time(dt: T.Union[datetime.datetime, int, float]) -> float:
|
|
190
136
|
if isinstance(dt, (int, float)):
|
|
191
137
|
return dt
|
|
@@ -7,7 +7,7 @@ import typing as T
|
|
|
7
7
|
import pynmea2
|
|
8
8
|
|
|
9
9
|
from .. import geo
|
|
10
|
-
from
|
|
10
|
+
from ..mp4 import simple_mp4_parser as sparser
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
LOG = logging.getLogger(__name__)
|
|
@@ -55,8 +55,8 @@ def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
|
|
|
55
55
|
|
|
56
56
|
def extract_camera_model(fp: T.BinaryIO) -> str:
|
|
57
57
|
try:
|
|
58
|
-
cprt_bytes =
|
|
59
|
-
except
|
|
58
|
+
cprt_bytes = sparser.parse_mp4_data_first(fp, [b"free", b"cprt"])
|
|
59
|
+
except sparser.ParsingError:
|
|
60
60
|
return ""
|
|
61
61
|
|
|
62
62
|
if cprt_bytes is None:
|
|
@@ -91,7 +91,7 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
|
|
|
91
91
|
|
|
92
92
|
|
|
93
93
|
def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
|
|
94
|
-
gps_data =
|
|
94
|
+
gps_data = sparser.parse_mp4_data_first(fp, [b"free", b"gps "])
|
|
95
95
|
if gps_data is None:
|
|
96
96
|
return None
|
|
97
97
|
|
|
@@ -6,7 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
|
|
7
7
|
from tqdm import tqdm
|
|
8
8
|
|
|
9
|
-
from .. import exceptions, exif_write, geo, types
|
|
9
|
+
from .. import exceptions, exif_write, geo, types, utils
|
|
10
10
|
from ..exif_read import ExifRead, ExifReadABC
|
|
11
11
|
from .geotag_from_generic import GeotagImagesFromGeneric
|
|
12
12
|
|
|
@@ -64,6 +64,7 @@ class GeotagImagesFromEXIF(GeotagImagesFromGeneric):
|
|
|
64
64
|
image_metadata = types.ImageMetadata(
|
|
65
65
|
filename=image_path,
|
|
66
66
|
md5sum=None,
|
|
67
|
+
filesize=utils.get_file_size(image_path),
|
|
67
68
|
time=geo.as_unix_time(capture_time),
|
|
68
69
|
lat=lat,
|
|
69
70
|
lon=lon,
|
|
@@ -48,7 +48,6 @@ class GeotagImagesFromExifToolBothImageAndVideo(GeotagImagesFromGeneric):
|
|
|
48
48
|
video_paths = utils.find_videos(
|
|
49
49
|
[Path(pathstr) for pathstr in rdf_description_by_path.keys()],
|
|
50
50
|
skip_subfolders=True,
|
|
51
|
-
check_file_suffix=True,
|
|
52
51
|
)
|
|
53
52
|
|
|
54
53
|
# will try to geotag these error metadatas from video later
|