mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapillary_tools/__init__.py +1 -1
- mapillary_tools/api_v4.py +4 -4
- mapillary_tools/camm/camm_parser.py +5 -5
- mapillary_tools/commands/__main__.py +1 -2
- mapillary_tools/config.py +7 -5
- mapillary_tools/constants.py +1 -2
- mapillary_tools/exceptions.py +1 -1
- mapillary_tools/exif_read.py +65 -65
- mapillary_tools/exif_write.py +7 -7
- mapillary_tools/exiftool_read.py +23 -46
- mapillary_tools/exiftool_read_video.py +36 -34
- mapillary_tools/ffmpeg.py +24 -23
- mapillary_tools/geo.py +4 -21
- mapillary_tools/geotag/{geotag_from_generic.py → base.py} +32 -48
- mapillary_tools/geotag/factory.py +27 -34
- mapillary_tools/geotag/geotag_images_from_exif.py +15 -51
- mapillary_tools/geotag/geotag_images_from_exiftool.py +107 -59
- mapillary_tools/geotag/geotag_images_from_gpx.py +20 -10
- mapillary_tools/geotag/geotag_images_from_gpx_file.py +2 -34
- mapillary_tools/geotag/geotag_images_from_nmea_file.py +0 -3
- mapillary_tools/geotag/geotag_images_from_video.py +16 -14
- mapillary_tools/geotag/geotag_videos_from_exiftool.py +97 -0
- mapillary_tools/geotag/geotag_videos_from_gpx.py +14 -115
- mapillary_tools/geotag/geotag_videos_from_video.py +14 -147
- mapillary_tools/geotag/image_extractors/base.py +18 -0
- mapillary_tools/geotag/image_extractors/exif.py +60 -0
- mapillary_tools/geotag/image_extractors/exiftool.py +18 -0
- mapillary_tools/geotag/options.py +1 -0
- mapillary_tools/geotag/utils.py +62 -0
- mapillary_tools/geotag/video_extractors/base.py +18 -0
- mapillary_tools/geotag/video_extractors/exiftool.py +70 -0
- mapillary_tools/{video_data_extraction/extractors/gpx_parser.py → geotag/video_extractors/gpx.py} +57 -39
- mapillary_tools/geotag/video_extractors/native.py +157 -0
- mapillary_tools/gpmf/gpmf_parser.py +16 -16
- mapillary_tools/gpmf/gps_filter.py +5 -3
- mapillary_tools/history.py +4 -2
- mapillary_tools/mp4/construct_mp4_parser.py +9 -8
- mapillary_tools/mp4/mp4_sample_parser.py +27 -27
- mapillary_tools/mp4/simple_mp4_builder.py +10 -9
- mapillary_tools/mp4/simple_mp4_parser.py +13 -12
- mapillary_tools/process_geotag_properties.py +5 -7
- mapillary_tools/process_sequence_properties.py +40 -38
- mapillary_tools/sample_video.py +8 -8
- mapillary_tools/telemetry.py +6 -5
- mapillary_tools/types.py +33 -38
- mapillary_tools/utils.py +16 -18
- {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/METADATA +1 -1
- mapillary_tools-0.14.0a2.dist-info/RECORD +72 -0
- mapillary_tools/geotag/__init__.py +0 -1
- mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -77
- mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +0 -151
- mapillary_tools/video_data_extraction/cli_options.py +0 -22
- mapillary_tools/video_data_extraction/extract_video_data.py +0 -157
- mapillary_tools/video_data_extraction/extractors/base_parser.py +0 -75
- mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +0 -49
- mapillary_tools/video_data_extraction/extractors/camm_parser.py +0 -62
- mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +0 -74
- mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +0 -52
- mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +0 -52
- mapillary_tools/video_data_extraction/extractors/gopro_parser.py +0 -58
- mapillary_tools/video_data_extraction/extractors/nmea_parser.py +0 -24
- mapillary_tools/video_data_extraction/video_data_parser_factory.py +0 -39
- mapillary_tools-0.14.0a1.dist-info/RECORD +0 -78
- {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/WHEEL +0 -0
- {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/entry_points.txt +0 -0
- {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/licenses/LICENSE +0 -0
- {mapillary_tools-0.14.0a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/top_level.txt +0 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import statistics
|
|
2
4
|
import typing as T
|
|
3
5
|
|
|
@@ -96,7 +98,7 @@ def both(
|
|
|
96
98
|
def dbscan(
|
|
97
99
|
sequences: T.Sequence[PointSequence],
|
|
98
100
|
merge_or_not: Decider,
|
|
99
|
-
) ->
|
|
101
|
+
) -> dict[int, PointSequence]:
|
|
100
102
|
"""
|
|
101
103
|
One-dimension DBSCAN clustering: https://en.wikipedia.org/wiki/DBSCAN
|
|
102
104
|
The input is a list of sequences, and it is guaranteed that all sequences are sorted by time.
|
|
@@ -107,7 +109,7 @@ def dbscan(
|
|
|
107
109
|
"""
|
|
108
110
|
|
|
109
111
|
# find which sequences (keys) should be merged to which sequences (values)
|
|
110
|
-
mergeto:
|
|
112
|
+
mergeto: dict[int, int] = {}
|
|
111
113
|
for left in range(len(sequences)):
|
|
112
114
|
mergeto.setdefault(left, left)
|
|
113
115
|
# find the first sequence to merge with
|
|
@@ -119,7 +121,7 @@ def dbscan(
|
|
|
119
121
|
break
|
|
120
122
|
|
|
121
123
|
# merge
|
|
122
|
-
merged:
|
|
124
|
+
merged: dict[int, PointSequence] = {}
|
|
123
125
|
for idx, s in enumerate(sequences):
|
|
124
126
|
merged.setdefault(mergeto[idx], []).extend(s)
|
|
125
127
|
|
mapillary_tools/history.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
import logging
|
|
3
5
|
import string
|
|
@@ -43,14 +45,14 @@ def write_history(
|
|
|
43
45
|
md5sum: str,
|
|
44
46
|
params: JSONDict,
|
|
45
47
|
summary: JSONDict,
|
|
46
|
-
metadatas: T.
|
|
48
|
+
metadatas: T.Sequence[types.Metadata] | None = None,
|
|
47
49
|
) -> None:
|
|
48
50
|
if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
|
|
49
51
|
return
|
|
50
52
|
path = history_desc_path(md5sum)
|
|
51
53
|
LOG.debug("Writing upload history: %s", path)
|
|
52
54
|
path.resolve().parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
-
history:
|
|
55
|
+
history: dict[str, T.Any] = {
|
|
54
56
|
"params": params,
|
|
55
57
|
"summary": summary,
|
|
56
58
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# pyre-ignore-all-errors[5, 16, 21, 58]
|
|
2
|
+
from __future__ import annotations
|
|
2
3
|
|
|
3
4
|
import typing as T
|
|
4
5
|
|
|
@@ -42,7 +43,7 @@ BoxType = T.Literal[
|
|
|
42
43
|
|
|
43
44
|
class BoxDict(T.TypedDict, total=True):
|
|
44
45
|
type: BoxType
|
|
45
|
-
data: T.
|
|
46
|
+
data: T.Sequence["BoxDict"] | dict[str, T.Any] | bytes
|
|
46
47
|
|
|
47
48
|
|
|
48
49
|
_UNITY_MATRIX = [0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000]
|
|
@@ -376,7 +377,7 @@ class Box64ConstructBuilder:
|
|
|
376
377
|
NOTE: Do not build data with this struct. For building, use Box32StructBuilder instead.
|
|
377
378
|
"""
|
|
378
379
|
|
|
379
|
-
_box:
|
|
380
|
+
_box: C.Construct | None
|
|
380
381
|
|
|
381
382
|
def __init__(
|
|
382
383
|
self,
|
|
@@ -438,7 +439,7 @@ class Box64ConstructBuilder:
|
|
|
438
439
|
def parse_box(self, data: bytes) -> BoxDict:
|
|
439
440
|
return T.cast(BoxDict, self.Box.parse(data))
|
|
440
441
|
|
|
441
|
-
def parse_boxlist(self, data: bytes) ->
|
|
442
|
+
def parse_boxlist(self, data: bytes) -> list[BoxDict]:
|
|
442
443
|
return T.cast(T.List[BoxDict], self.BoxList.parse(data))
|
|
443
444
|
|
|
444
445
|
|
|
@@ -464,7 +465,7 @@ class Box32ConstructBuilder(Box64ConstructBuilder):
|
|
|
464
465
|
def parse_box(self, data: bytes) -> BoxDict:
|
|
465
466
|
raise NotImplementedError("Box32ConstructBuilder does not support parsing")
|
|
466
467
|
|
|
467
|
-
def parse_boxlist(self, data: bytes) ->
|
|
468
|
+
def parse_boxlist(self, data: bytes) -> list[BoxDict]:
|
|
468
469
|
raise NotImplementedError("Box32ConstructBuilder does not support parsing")
|
|
469
470
|
|
|
470
471
|
def build_box(self, box: BoxDict) -> bytes:
|
|
@@ -584,7 +585,7 @@ MOOVWithoutSTBLBuilderConstruct = Box32ConstructBuilder(
|
|
|
584
585
|
|
|
585
586
|
|
|
586
587
|
def find_box_at_pathx(
|
|
587
|
-
box: T.
|
|
588
|
+
box: T.Sequence[BoxDict] | BoxDict, path: T.Sequence[bytes]
|
|
588
589
|
) -> BoxDict:
|
|
589
590
|
found = find_box_at_path(box, path)
|
|
590
591
|
if found is None:
|
|
@@ -593,8 +594,8 @@ def find_box_at_pathx(
|
|
|
593
594
|
|
|
594
595
|
|
|
595
596
|
def find_box_at_path(
|
|
596
|
-
box: T.
|
|
597
|
-
) ->
|
|
597
|
+
box: T.Sequence[BoxDict] | BoxDict, path: T.Sequence[bytes]
|
|
598
|
+
) -> BoxDict | None:
|
|
598
599
|
if not path:
|
|
599
600
|
return None
|
|
600
601
|
|
|
@@ -608,7 +609,7 @@ def find_box_at_path(
|
|
|
608
609
|
if box["type"] == path[0]:
|
|
609
610
|
if len(path) == 1:
|
|
610
611
|
return box
|
|
611
|
-
box_data = T.cast(T.
|
|
612
|
+
box_data = T.cast(T.List[BoxDict], box["data"])
|
|
612
613
|
# ListContainer from construct is not sequence
|
|
613
614
|
assert isinstance(box_data, T.Sequence), (
|
|
614
615
|
f"expect a list of boxes but got {type(box_data)} at path {path}"
|
|
@@ -44,16 +44,16 @@ class Sample(T.NamedTuple):
|
|
|
44
44
|
exact_timedelta: float
|
|
45
45
|
|
|
46
46
|
# reference to the sample description
|
|
47
|
-
description:
|
|
47
|
+
description: dict
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def _extract_raw_samples(
|
|
51
51
|
sizes: T.Sequence[int],
|
|
52
|
-
chunk_entries: T.Sequence[
|
|
52
|
+
chunk_entries: T.Sequence[dict],
|
|
53
53
|
chunk_offsets: T.Sequence[int],
|
|
54
54
|
timedeltas: T.Sequence[int],
|
|
55
|
-
composition_offsets:
|
|
56
|
-
syncs:
|
|
55
|
+
composition_offsets: list[int] | None,
|
|
56
|
+
syncs: set[int] | None,
|
|
57
57
|
) -> T.Generator[RawSample, None, None]:
|
|
58
58
|
if not sizes:
|
|
59
59
|
return
|
|
@@ -130,7 +130,7 @@ def _extract_raw_samples(
|
|
|
130
130
|
|
|
131
131
|
def _extract_samples(
|
|
132
132
|
raw_samples: T.Iterator[RawSample],
|
|
133
|
-
descriptions:
|
|
133
|
+
descriptions: list,
|
|
134
134
|
timescale: int,
|
|
135
135
|
) -> T.Generator[Sample, None, None]:
|
|
136
136
|
acc_delta = 0
|
|
@@ -154,21 +154,21 @@ STBLBoxlistConstruct = cparser.Box64ConstructBuilder(
|
|
|
154
154
|
|
|
155
155
|
def extract_raw_samples_from_stbl_data(
|
|
156
156
|
stbl: bytes,
|
|
157
|
-
) ->
|
|
158
|
-
descriptions = []
|
|
159
|
-
sizes = []
|
|
160
|
-
chunk_offsets = []
|
|
161
|
-
chunk_entries = []
|
|
162
|
-
timedeltas:
|
|
163
|
-
composition_offsets:
|
|
164
|
-
syncs:
|
|
157
|
+
) -> tuple[list[dict], T.Generator[RawSample, None, None]]:
|
|
158
|
+
descriptions: list[dict] = []
|
|
159
|
+
sizes: list[int] = []
|
|
160
|
+
chunk_offsets: list[int] = []
|
|
161
|
+
chunk_entries: list[dict] = []
|
|
162
|
+
timedeltas: list[int] = []
|
|
163
|
+
composition_offsets: list[int] | None = None
|
|
164
|
+
syncs: set[int] | None = None
|
|
165
165
|
|
|
166
166
|
stbl_children = T.cast(
|
|
167
167
|
T.Sequence[cparser.BoxDict], STBLBoxlistConstruct.parse(stbl)
|
|
168
168
|
)
|
|
169
169
|
|
|
170
170
|
for box in stbl_children:
|
|
171
|
-
data:
|
|
171
|
+
data: dict = T.cast(dict, box["data"])
|
|
172
172
|
|
|
173
173
|
if box["type"] == b"stsd":
|
|
174
174
|
descriptions = list(data["entries"])
|
|
@@ -227,32 +227,32 @@ class TrackBoxParser:
|
|
|
227
227
|
)
|
|
228
228
|
self.stbl_data = T.cast(bytes, stbl["data"])
|
|
229
229
|
|
|
230
|
-
def extract_tkhd_boxdata(self) ->
|
|
230
|
+
def extract_tkhd_boxdata(self) -> dict:
|
|
231
231
|
return T.cast(
|
|
232
|
-
|
|
232
|
+
dict, cparser.find_box_at_pathx(self.trak_children, [b"tkhd"])["data"]
|
|
233
233
|
)
|
|
234
234
|
|
|
235
235
|
def is_video_track(self) -> bool:
|
|
236
236
|
hdlr = cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"hdlr"])
|
|
237
237
|
return T.cast(T.Dict[str, T.Any], hdlr["data"])["handler_type"] == b"vide"
|
|
238
238
|
|
|
239
|
-
def extract_sample_descriptions(self) ->
|
|
239
|
+
def extract_sample_descriptions(self) -> list[dict]:
|
|
240
240
|
# TODO: return [] if parsing fail
|
|
241
241
|
boxes = _STSDBoxListConstruct.parse(self.stbl_data)
|
|
242
242
|
stsd = cparser.find_box_at_pathx(
|
|
243
243
|
T.cast(T.Sequence[cparser.BoxDict], boxes), [b"stsd"]
|
|
244
244
|
)
|
|
245
|
-
return T.cast(T.List[
|
|
245
|
+
return T.cast(T.List[dict], T.cast(dict, stsd["data"])["entries"])
|
|
246
246
|
|
|
247
|
-
def extract_elst_boxdata(self) ->
|
|
247
|
+
def extract_elst_boxdata(self) -> dict | None:
|
|
248
248
|
box = cparser.find_box_at_path(self.trak_children, [b"edts", b"elst"])
|
|
249
249
|
if box is None:
|
|
250
250
|
return None
|
|
251
|
-
return T.cast(
|
|
251
|
+
return T.cast(dict, box["data"])
|
|
252
252
|
|
|
253
|
-
def extract_mdhd_boxdata(self) ->
|
|
253
|
+
def extract_mdhd_boxdata(self) -> dict:
|
|
254
254
|
box = cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"mdhd"])
|
|
255
|
-
return T.cast(
|
|
255
|
+
return T.cast(dict, box["data"])
|
|
256
256
|
|
|
257
257
|
def extract_raw_samples(self) -> T.Generator[RawSample, None, None]:
|
|
258
258
|
_, raw_samples = extract_raw_samples_from_stbl_data(self.stbl_data)
|
|
@@ -261,7 +261,7 @@ class TrackBoxParser:
|
|
|
261
261
|
def extract_samples(self) -> T.Generator[Sample, None, None]:
|
|
262
262
|
descriptions, raw_samples = extract_raw_samples_from_stbl_data(self.stbl_data)
|
|
263
263
|
mdhd = T.cast(
|
|
264
|
-
|
|
264
|
+
dict,
|
|
265
265
|
cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"mdhd"])["data"],
|
|
266
266
|
)
|
|
267
267
|
yield from _extract_samples(raw_samples, descriptions, mdhd["timescale"])
|
|
@@ -287,15 +287,15 @@ class MovieBoxParser:
|
|
|
287
287
|
moov = sparser.parse_box_data_firstx(stream, [b"moov"])
|
|
288
288
|
return cls(moov)
|
|
289
289
|
|
|
290
|
-
def extract_mvhd_boxdata(self) ->
|
|
290
|
+
def extract_mvhd_boxdata(self) -> dict:
|
|
291
291
|
mvhd = cparser.find_box_at_pathx(self.moov_children, [b"mvhd"])
|
|
292
|
-
return T.cast(
|
|
292
|
+
return T.cast(dict, mvhd["data"])
|
|
293
293
|
|
|
294
|
-
def extract_udta_boxdata(self) ->
|
|
294
|
+
def extract_udta_boxdata(self) -> dict | None:
|
|
295
295
|
box = cparser.find_box_at_path(self.moov_children, [b"udta"])
|
|
296
296
|
if box is None:
|
|
297
297
|
return None
|
|
298
|
-
return T.cast(
|
|
298
|
+
return T.cast(dict, box["data"])
|
|
299
299
|
|
|
300
300
|
def extract_tracks(self) -> T.Generator[TrackBoxParser, None, None]:
|
|
301
301
|
for box in self.moov_children:
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import dataclasses
|
|
2
4
|
import io
|
|
3
5
|
import typing as T
|
|
@@ -64,8 +66,8 @@ class _SampleChunk:
|
|
|
64
66
|
offset: int
|
|
65
67
|
|
|
66
68
|
|
|
67
|
-
def _build_chunks(raw_samples: T.Iterable[RawSample]) ->
|
|
68
|
-
chunks:
|
|
69
|
+
def _build_chunks(raw_samples: T.Iterable[RawSample]) -> list[_SampleChunk]:
|
|
70
|
+
chunks: list[_SampleChunk] = []
|
|
69
71
|
prev_raw_sample = None
|
|
70
72
|
|
|
71
73
|
for raw_sample in raw_samples:
|
|
@@ -120,7 +122,7 @@ class _CompressedSampleDelta:
|
|
|
120
122
|
|
|
121
123
|
def _build_stts(sample_deltas: T.Iterable[int]) -> BoxDict:
|
|
122
124
|
# compress deltas
|
|
123
|
-
compressed:
|
|
125
|
+
compressed: list[_CompressedSampleDelta] = []
|
|
124
126
|
for delta in sample_deltas:
|
|
125
127
|
if compressed and delta == compressed[-1].sample_delta:
|
|
126
128
|
compressed[-1].sample_count += 1
|
|
@@ -146,7 +148,7 @@ class _CompressedSampleCompositionOffset:
|
|
|
146
148
|
|
|
147
149
|
def _build_ctts(sample_composition_offsets: T.Iterable[int]) -> BoxDict:
|
|
148
150
|
# compress offsets
|
|
149
|
-
compressed:
|
|
151
|
+
compressed: list[_CompressedSampleCompositionOffset] = []
|
|
150
152
|
for offset in sample_composition_offsets:
|
|
151
153
|
if compressed and offset == compressed[-1].sample_offset:
|
|
152
154
|
compressed[-1].sample_count += 1
|
|
@@ -182,7 +184,7 @@ def _build_stss(is_syncs: T.Iterable[bool]) -> BoxDict:
|
|
|
182
184
|
|
|
183
185
|
def build_stbl_from_raw_samples(
|
|
184
186
|
descriptions: T.Sequence[T.Any], raw_samples: T.Iterable[RawSample]
|
|
185
|
-
) ->
|
|
187
|
+
) -> list[BoxDict]:
|
|
186
188
|
# raw_samples could be iterator so convert to list
|
|
187
189
|
raw_samples = list(raw_samples)
|
|
188
190
|
# It is recommended that the boxes within the Sample Table Box be in the following order:
|
|
@@ -329,9 +331,8 @@ _MOOVChildrenParserConstruct = cparser.Box64ConstructBuilder(
|
|
|
329
331
|
|
|
330
332
|
def transform_mp4(
|
|
331
333
|
src_fp: T.BinaryIO,
|
|
332
|
-
sample_generator: T.
|
|
333
|
-
|
|
334
|
-
] = None,
|
|
334
|
+
sample_generator: T.Callable[[T.BinaryIO, list[BoxDict]], T.Iterator[io.IOBase]]
|
|
335
|
+
| None = None,
|
|
335
336
|
) -> io_utils.ChainedIO:
|
|
336
337
|
# extract ftyp
|
|
337
338
|
src_fp.seek(0)
|
|
@@ -347,7 +348,7 @@ def transform_mp4(
|
|
|
347
348
|
|
|
348
349
|
# extract video samples
|
|
349
350
|
source_samples = list(iterate_samples(moov_children))
|
|
350
|
-
sample_readers:
|
|
351
|
+
sample_readers: list[io.IOBase] = [
|
|
351
352
|
io_utils.SlicedIO(src_fp, sample.offset, sample.size)
|
|
352
353
|
for sample in source_samples
|
|
353
354
|
]
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# pyre-ignore-all-errors[5, 16, 21, 24, 58]
|
|
2
|
+
from __future__ import annotations
|
|
2
3
|
|
|
3
4
|
import io
|
|
4
5
|
import typing as T
|
|
@@ -130,8 +131,8 @@ def parse_boxes_recursive(
|
|
|
130
131
|
stream: T.BinaryIO,
|
|
131
132
|
maxsize: int = -1,
|
|
132
133
|
depth: int = 0,
|
|
133
|
-
box_list_types:
|
|
134
|
-
) -> T.Generator[
|
|
134
|
+
box_list_types: set[bytes] | None = None,
|
|
135
|
+
) -> T.Generator[tuple[Header, int, T.BinaryIO], None, None]:
|
|
135
136
|
assert maxsize == -1 or 0 <= maxsize
|
|
136
137
|
|
|
137
138
|
if box_list_types is None:
|
|
@@ -152,10 +153,10 @@ def parse_boxes_recursive(
|
|
|
152
153
|
|
|
153
154
|
def parse_path(
|
|
154
155
|
stream: T.BinaryIO,
|
|
155
|
-
path: T.Sequence[
|
|
156
|
+
path: T.Sequence[bytes | T.Sequence[bytes]],
|
|
156
157
|
maxsize: int = -1,
|
|
157
158
|
depth: int = 0,
|
|
158
|
-
) -> T.Generator[
|
|
159
|
+
) -> T.Generator[tuple[Header, T.BinaryIO], None, None]:
|
|
159
160
|
if not path:
|
|
160
161
|
return
|
|
161
162
|
|
|
@@ -172,8 +173,8 @@ def parse_path(
|
|
|
172
173
|
|
|
173
174
|
|
|
174
175
|
def _parse_path_first(
|
|
175
|
-
stream: T.BinaryIO, path:
|
|
176
|
-
) ->
|
|
176
|
+
stream: T.BinaryIO, path: list[bytes], maxsize: int = -1, depth: int = 0
|
|
177
|
+
) -> tuple[Header, T.BinaryIO] | None:
|
|
177
178
|
if not path:
|
|
178
179
|
return None
|
|
179
180
|
for h, s in parse_boxes(stream, maxsize=maxsize, extend_eof=depth == 0):
|
|
@@ -188,8 +189,8 @@ def _parse_path_first(
|
|
|
188
189
|
|
|
189
190
|
|
|
190
191
|
def parse_mp4_data_first(
|
|
191
|
-
stream: T.BinaryIO, path:
|
|
192
|
-
) ->
|
|
192
|
+
stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
|
|
193
|
+
) -> bytes | None:
|
|
193
194
|
# depth=0 will enable EoF extension
|
|
194
195
|
parsed = _parse_path_first(stream, path, maxsize=maxsize, depth=0)
|
|
195
196
|
if parsed is None:
|
|
@@ -199,7 +200,7 @@ def parse_mp4_data_first(
|
|
|
199
200
|
|
|
200
201
|
|
|
201
202
|
def parse_mp4_data_firstx(
|
|
202
|
-
stream: T.BinaryIO, path:
|
|
203
|
+
stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
|
|
203
204
|
) -> bytes:
|
|
204
205
|
data = parse_mp4_data_first(stream, path, maxsize=maxsize)
|
|
205
206
|
if data is None:
|
|
@@ -208,8 +209,8 @@ def parse_mp4_data_firstx(
|
|
|
208
209
|
|
|
209
210
|
|
|
210
211
|
def parse_box_data_first(
|
|
211
|
-
stream: T.BinaryIO, path:
|
|
212
|
-
) ->
|
|
212
|
+
stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
|
|
213
|
+
) -> bytes | None:
|
|
213
214
|
# depth=1 will disable EoF extension
|
|
214
215
|
parsed = _parse_path_first(stream, path, maxsize=maxsize, depth=1)
|
|
215
216
|
if parsed is None:
|
|
@@ -219,7 +220,7 @@ def parse_box_data_first(
|
|
|
219
220
|
|
|
220
221
|
|
|
221
222
|
def parse_box_data_firstx(
|
|
222
|
-
stream: T.BinaryIO, path:
|
|
223
|
+
stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
|
|
223
224
|
) -> bytes:
|
|
224
225
|
data = parse_box_data_first(stream, path, maxsize=maxsize)
|
|
225
226
|
if data is None:
|
|
@@ -25,9 +25,7 @@ DEFAULT_GEOTAG_SOURCE_OPTIONS = [
|
|
|
25
25
|
]
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
def _normalize_import_paths(
|
|
29
|
-
import_path: T.Union[Path, T.Sequence[Path]],
|
|
30
|
-
) -> T.Sequence[Path]:
|
|
28
|
+
def _normalize_import_paths(import_path: Path | T.Sequence[Path]) -> T.Sequence[Path]:
|
|
31
29
|
import_paths: T.Sequence[Path]
|
|
32
30
|
if isinstance(import_path, Path):
|
|
33
31
|
import_paths = [import_path]
|
|
@@ -219,7 +217,7 @@ def _is_error_skipped(error_type: str, skipped_process_errors: set[T.Type[Except
|
|
|
219
217
|
|
|
220
218
|
def _show_stats(
|
|
221
219
|
metadatas: T.Sequence[types.MetadataOrError],
|
|
222
|
-
skipped_process_errors:
|
|
220
|
+
skipped_process_errors: set[T.Type[Exception]],
|
|
223
221
|
) -> None:
|
|
224
222
|
metadatas_by_filetype: dict[types.FileType, list[types.MetadataOrError]] = {}
|
|
225
223
|
for metadata in metadatas:
|
|
@@ -249,7 +247,7 @@ def _show_stats(
|
|
|
249
247
|
def _show_stats_per_filetype(
|
|
250
248
|
metadatas: T.Collection[types.MetadataOrError],
|
|
251
249
|
filetype: types.FileType,
|
|
252
|
-
skipped_process_errors:
|
|
250
|
+
skipped_process_errors: set[T.Type[Exception]],
|
|
253
251
|
):
|
|
254
252
|
good_metadatas: list[types.Metadata]
|
|
255
253
|
good_metadatas, error_metadatas = types.separate_errors(metadatas)
|
|
@@ -314,7 +312,7 @@ def _validate_metadatas(
|
|
|
314
312
|
|
|
315
313
|
|
|
316
314
|
def process_finalize(
|
|
317
|
-
import_path: T.
|
|
315
|
+
import_path: T.Sequence[Path] | Path,
|
|
318
316
|
metadatas: list[types.MetadataOrError],
|
|
319
317
|
skip_process_errors: bool = False,
|
|
320
318
|
device_make: str | None = None,
|
|
@@ -406,7 +404,7 @@ def process_finalize(
|
|
|
406
404
|
_write_metadatas(metadatas, desc_path)
|
|
407
405
|
|
|
408
406
|
# Show stats
|
|
409
|
-
skipped_process_errors:
|
|
407
|
+
skipped_process_errors: set[T.Type[Exception]]
|
|
410
408
|
if skip_process_errors:
|
|
411
409
|
# Skip all exceptions
|
|
412
410
|
skipped_process_errors = {Exception}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import itertools
|
|
2
4
|
import logging
|
|
3
5
|
import math
|
|
@@ -14,13 +16,13 @@ PointSequence = T.List[geo.PointLike]
|
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
def split_sequence_by(
|
|
17
|
-
sequence: T.
|
|
19
|
+
sequence: T.Sequence[SeqItem],
|
|
18
20
|
should_split: T.Callable[[SeqItem, SeqItem], bool],
|
|
19
|
-
) ->
|
|
21
|
+
) -> list[list[SeqItem]]:
|
|
20
22
|
"""
|
|
21
23
|
Split a sequence into multiple sequences by should_split(prev, cur) => True
|
|
22
24
|
"""
|
|
23
|
-
output_sequences:
|
|
25
|
+
output_sequences: list[list[SeqItem]] = []
|
|
24
26
|
|
|
25
27
|
seq = iter(sequence)
|
|
26
28
|
|
|
@@ -45,14 +47,14 @@ def split_sequence_by(
|
|
|
45
47
|
|
|
46
48
|
|
|
47
49
|
def split_sequence_by_agg(
|
|
48
|
-
sequence: T.
|
|
49
|
-
should_split_with_sequence_state: T.Callable[[SeqItem,
|
|
50
|
-
) ->
|
|
50
|
+
sequence: T.Sequence[SeqItem],
|
|
51
|
+
should_split_with_sequence_state: T.Callable[[SeqItem, dict], bool],
|
|
52
|
+
) -> list[list[SeqItem]]:
|
|
51
53
|
"""
|
|
52
54
|
Split a sequence by should_split_with_sequence_state(cur, sequence_state) => True
|
|
53
55
|
"""
|
|
54
|
-
output_sequences:
|
|
55
|
-
sequence_state:
|
|
56
|
+
output_sequences: list[list[SeqItem]] = []
|
|
57
|
+
sequence_state: dict = {}
|
|
56
58
|
|
|
57
59
|
for cur in sequence:
|
|
58
60
|
start_new_sequence = should_split_with_sequence_state(cur, sequence_state)
|
|
@@ -77,9 +79,9 @@ def duplication_check(
|
|
|
77
79
|
sequence: PointSequence,
|
|
78
80
|
max_duplicate_distance: float,
|
|
79
81
|
max_duplicate_angle: float,
|
|
80
|
-
) ->
|
|
82
|
+
) -> tuple[PointSequence, list[types.ErrorMetadata]]:
|
|
81
83
|
dedups: PointSequence = []
|
|
82
|
-
dups:
|
|
84
|
+
dups: list[types.ErrorMetadata] = []
|
|
83
85
|
|
|
84
86
|
it = iter(sequence)
|
|
85
87
|
prev = next(it)
|
|
@@ -125,10 +127,10 @@ def duplication_check(
|
|
|
125
127
|
|
|
126
128
|
|
|
127
129
|
def _group_by(
|
|
128
|
-
image_metadatas: T.
|
|
130
|
+
image_metadatas: T.Iterable[types.ImageMetadata],
|
|
129
131
|
group_key_func=T.Callable[[types.ImageMetadata], T.Hashable],
|
|
130
|
-
) ->
|
|
131
|
-
grouped:
|
|
132
|
+
) -> dict[T.Hashable, list[types.ImageMetadata]]:
|
|
133
|
+
grouped: dict[T.Hashable, list[types.ImageMetadata]] = {}
|
|
132
134
|
for metadata in image_metadatas:
|
|
133
135
|
grouped.setdefault(group_key_func(metadata), []).append(metadata)
|
|
134
136
|
return grouped
|
|
@@ -245,13 +247,13 @@ def _is_video_stationary(
|
|
|
245
247
|
|
|
246
248
|
|
|
247
249
|
def _check_video_limits(
|
|
248
|
-
video_metadatas: T.
|
|
250
|
+
video_metadatas: T.Iterable[types.VideoMetadata],
|
|
249
251
|
max_sequence_filesize_in_bytes: int,
|
|
250
252
|
max_avg_speed: float,
|
|
251
253
|
max_radius_for_stationary_check: float,
|
|
252
|
-
) ->
|
|
253
|
-
output_video_metadatas:
|
|
254
|
-
error_metadatas:
|
|
254
|
+
) -> tuple[list[types.VideoMetadata], list[types.ErrorMetadata]]:
|
|
255
|
+
output_video_metadatas: list[types.VideoMetadata] = []
|
|
256
|
+
error_metadatas: list[types.ErrorMetadata] = []
|
|
255
257
|
|
|
256
258
|
for video_metadata in video_metadatas:
|
|
257
259
|
try:
|
|
@@ -312,9 +314,9 @@ def _check_sequences_by_limits(
|
|
|
312
314
|
input_sequences: T.Sequence[PointSequence],
|
|
313
315
|
max_sequence_filesize_in_bytes: int,
|
|
314
316
|
max_avg_speed: float,
|
|
315
|
-
) ->
|
|
316
|
-
output_sequences:
|
|
317
|
-
output_errors:
|
|
317
|
+
) -> tuple[list[PointSequence], list[types.ErrorMetadata]]:
|
|
318
|
+
output_sequences: list[PointSequence] = []
|
|
319
|
+
output_errors: list[types.ErrorMetadata] = []
|
|
318
320
|
|
|
319
321
|
for sequence in input_sequences:
|
|
320
322
|
sequence_filesize = sum(
|
|
@@ -370,8 +372,8 @@ def _check_sequences_by_limits(
|
|
|
370
372
|
|
|
371
373
|
|
|
372
374
|
def _group_by_folder_and_camera(
|
|
373
|
-
image_metadatas:
|
|
374
|
-
) ->
|
|
375
|
+
image_metadatas: list[types.ImageMetadata],
|
|
376
|
+
) -> list[list[types.ImageMetadata]]:
|
|
375
377
|
grouped = _group_by(
|
|
376
378
|
image_metadatas,
|
|
377
379
|
lambda metadata: (
|
|
@@ -395,8 +397,8 @@ def _group_by_folder_and_camera(
|
|
|
395
397
|
|
|
396
398
|
|
|
397
399
|
def _split_sequences_by_cutoff_time(
|
|
398
|
-
input_sequences: T.
|
|
399
|
-
) ->
|
|
400
|
+
input_sequences: T.Sequence[PointSequence], cutoff_time: float
|
|
401
|
+
) -> list[PointSequence]:
|
|
400
402
|
def _should_split_by_cutoff_time(
|
|
401
403
|
prev: types.ImageMetadata, cur: types.ImageMetadata
|
|
402
404
|
) -> bool:
|
|
@@ -432,8 +434,8 @@ def _split_sequences_by_cutoff_time(
|
|
|
432
434
|
|
|
433
435
|
|
|
434
436
|
def _split_sequences_by_cutoff_distance(
|
|
435
|
-
input_sequences: T.
|
|
436
|
-
) ->
|
|
437
|
+
input_sequences: T.Sequence[PointSequence], cutoff_distance: float
|
|
438
|
+
) -> list[PointSequence]:
|
|
437
439
|
def _should_split_by_cutoff_distance(
|
|
438
440
|
prev: types.ImageMetadata, cur: types.ImageMetadata
|
|
439
441
|
) -> bool:
|
|
@@ -471,12 +473,12 @@ def _split_sequences_by_cutoff_distance(
|
|
|
471
473
|
|
|
472
474
|
|
|
473
475
|
def _check_sequences_duplication(
|
|
474
|
-
input_sequences: T.
|
|
476
|
+
input_sequences: T.Sequence[PointSequence],
|
|
475
477
|
duplicate_distance: float,
|
|
476
478
|
duplicate_angle: float,
|
|
477
|
-
) ->
|
|
478
|
-
output_sequences:
|
|
479
|
-
output_errors:
|
|
479
|
+
) -> tuple[list[PointSequence], list[types.ErrorMetadata]]:
|
|
480
|
+
output_sequences: list[PointSequence] = []
|
|
481
|
+
output_errors: list[types.ErrorMetadata] = []
|
|
480
482
|
|
|
481
483
|
for sequence in input_sequences:
|
|
482
484
|
output_sequence, errors = duplication_check(
|
|
@@ -502,14 +504,14 @@ def _check_sequences_duplication(
|
|
|
502
504
|
|
|
503
505
|
|
|
504
506
|
def _split_sequences_by_limits(
|
|
505
|
-
input_sequences: T.
|
|
507
|
+
input_sequences: T.Sequence[PointSequence],
|
|
506
508
|
max_sequence_filesize_in_bytes: float,
|
|
507
509
|
max_sequence_pixels: float,
|
|
508
|
-
) ->
|
|
510
|
+
) -> list[PointSequence]:
|
|
509
511
|
max_sequence_images = constants.MAX_SEQUENCE_LENGTH
|
|
510
512
|
max_sequence_filesize = max_sequence_filesize_in_bytes
|
|
511
513
|
|
|
512
|
-
def _should_split(image: types.ImageMetadata, sequence_state:
|
|
514
|
+
def _should_split(image: types.ImageMetadata, sequence_state: dict) -> bool:
|
|
513
515
|
last_sequence_images = sequence_state.get("last_sequence_images", 0)
|
|
514
516
|
last_sequence_file_size = sequence_state.get("last_sequence_file_size", 0)
|
|
515
517
|
last_sequence_pixels = sequence_state.get("last_sequence_pixels", 0)
|
|
@@ -586,15 +588,15 @@ def process_sequence_properties(
|
|
|
586
588
|
duplicate_distance: float = constants.DUPLICATE_DISTANCE,
|
|
587
589
|
duplicate_angle: float = constants.DUPLICATE_ANGLE,
|
|
588
590
|
max_avg_speed: float = constants.MAX_AVG_SPEED,
|
|
589
|
-
) ->
|
|
591
|
+
) -> list[types.MetadataOrError]:
|
|
590
592
|
max_sequence_filesize_in_bytes = _parse_filesize_in_bytes(
|
|
591
593
|
constants.MAX_SEQUENCE_FILESIZE
|
|
592
594
|
)
|
|
593
595
|
max_sequence_pixels = _parse_pixels(constants.MAX_SEQUENCE_PIXELS)
|
|
594
596
|
|
|
595
|
-
error_metadatas:
|
|
596
|
-
image_metadatas:
|
|
597
|
-
video_metadatas:
|
|
597
|
+
error_metadatas: list[types.ErrorMetadata] = []
|
|
598
|
+
image_metadatas: list[types.ImageMetadata] = []
|
|
599
|
+
video_metadatas: list[types.VideoMetadata] = []
|
|
598
600
|
|
|
599
601
|
for metadata in metadatas:
|
|
600
602
|
if isinstance(metadata, types.ErrorMetadata):
|
|
@@ -617,7 +619,7 @@ def process_sequence_properties(
|
|
|
617
619
|
error_metadatas.extend(video_error_metadatas)
|
|
618
620
|
|
|
619
621
|
if image_metadatas:
|
|
620
|
-
sequences:
|
|
622
|
+
sequences: list[PointSequence]
|
|
621
623
|
|
|
622
624
|
# Group by folder and camera
|
|
623
625
|
sequences = _group_by_folder_and_camera(image_metadatas)
|