mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +94 -4
  3. mapillary_tools/{geotag → camm}/camm_builder.py +122 -62
  4. mapillary_tools/{geotag → camm}/camm_parser.py +120 -84
  5. mapillary_tools/commands/__init__.py +0 -1
  6. mapillary_tools/commands/__main__.py +0 -6
  7. mapillary_tools/commands/process.py +0 -50
  8. mapillary_tools/commands/upload.py +1 -26
  9. mapillary_tools/constants.py +2 -2
  10. mapillary_tools/exiftool_read_video.py +13 -11
  11. mapillary_tools/ffmpeg.py +2 -2
  12. mapillary_tools/geo.py +0 -54
  13. mapillary_tools/geotag/blackvue_parser.py +4 -4
  14. mapillary_tools/geotag/geotag_images_from_exif.py +2 -1
  15. mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -1
  16. mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +5 -3
  17. mapillary_tools/geotag/geotag_videos_from_video.py +13 -14
  18. mapillary_tools/geotag/gpmf_gps_filter.py +9 -10
  19. mapillary_tools/geotag/gpmf_parser.py +346 -83
  20. mapillary_tools/mp4/__init__.py +0 -0
  21. mapillary_tools/{geotag → mp4}/construct_mp4_parser.py +32 -16
  22. mapillary_tools/mp4/mp4_sample_parser.py +322 -0
  23. mapillary_tools/{geotag → mp4}/simple_mp4_builder.py +64 -38
  24. mapillary_tools/process_geotag_properties.py +25 -19
  25. mapillary_tools/process_sequence_properties.py +6 -6
  26. mapillary_tools/sample_video.py +17 -16
  27. mapillary_tools/telemetry.py +59 -0
  28. mapillary_tools/types.py +18 -0
  29. mapillary_tools/upload.py +75 -233
  30. mapillary_tools/upload_api_v4.py +8 -9
  31. mapillary_tools/utils.py +9 -16
  32. mapillary_tools/video_data_extraction/cli_options.py +0 -1
  33. mapillary_tools/video_data_extraction/extract_video_data.py +13 -31
  34. mapillary_tools/video_data_extraction/extractors/base_parser.py +13 -11
  35. mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +5 -4
  36. mapillary_tools/video_data_extraction/extractors/camm_parser.py +7 -6
  37. mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +4 -9
  38. mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +9 -11
  39. mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +6 -11
  40. mapillary_tools/video_data_extraction/extractors/gopro_parser.py +11 -4
  41. mapillary_tools/video_data_extraction/extractors/gpx_parser.py +54 -12
  42. mapillary_tools/video_data_extraction/extractors/nmea_parser.py +3 -3
  43. mapillary_tools/video_data_extraction/video_data_parser_factory.py +13 -20
  44. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/METADATA +10 -3
  45. mapillary_tools-0.13.1a1.dist-info/RECORD +75 -0
  46. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/WHEEL +1 -1
  47. mapillary_tools/commands/upload_blackvue.py +0 -33
  48. mapillary_tools/commands/upload_camm.py +0 -33
  49. mapillary_tools/commands/upload_zip.py +0 -33
  50. mapillary_tools/geotag/mp4_sample_parser.py +0 -426
  51. mapillary_tools/process_import_meta_properties.py +0 -76
  52. mapillary_tools-0.12.1.dist-info/RECORD +0 -77
  53. /mapillary_tools/{geotag → mp4}/io_utils.py +0 -0
  54. /mapillary_tools/{geotag → mp4}/simple_mp4_parser.py +0 -0
  55. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/LICENSE +0 -0
  56. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/entry_points.txt +0 -0
  57. {mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1a1.dist-info}/top_level.txt +0 -0
@@ -9,17 +9,22 @@ from enum import Enum
9
9
 
10
10
  import construct as C
11
11
 
12
- from . import (
13
- construct_mp4_parser as cparser,
14
- geo,
15
- mp4_sample_parser as sample_parser,
16
- simple_mp4_parser as parser,
17
- )
12
+ from .. import geo, telemetry
13
+ from ..mp4 import simple_mp4_parser as sparser
14
+ from ..mp4.mp4_sample_parser import MovieBoxParser, Sample, TrackBoxParser
18
15
 
19
16
 
20
17
  LOG = logging.getLogger(__name__)
21
18
 
22
19
 
20
+ TelemetryMeasurement = T.Union[
21
+ geo.Point,
22
+ telemetry.AccelerationData,
23
+ telemetry.GyroscopeData,
24
+ telemetry.MagnetometerData,
25
+ ]
26
+
27
+
23
28
  # Camera Motion Metadata Spec https://developers.google.com/streetview/publish/camm-spec
24
29
  class CAMMType(Enum):
25
30
  ANGLE_AXIS = 0
@@ -79,15 +84,15 @@ CAMMSampleData = C.Struct(
79
84
  )
80
85
 
81
86
 
82
- def _parse_point_from_sample(
83
- fp: T.BinaryIO, sample: sample_parser.Sample
84
- ) -> T.Optional[geo.Point]:
85
- fp.seek(sample.offset, io.SEEK_SET)
86
- data = fp.read(sample.size)
87
+ def _parse_telemetry_from_sample(
88
+ fp: T.BinaryIO, sample: Sample
89
+ ) -> T.Optional[TelemetryMeasurement]:
90
+ fp.seek(sample.raw_sample.offset, io.SEEK_SET)
91
+ data = fp.read(sample.raw_sample.size)
87
92
  box = CAMMSampleData.parse(data)
88
93
  if box.type == CAMMType.MIN_GPS.value:
89
94
  return geo.Point(
90
- time=sample.time_offset,
95
+ time=sample.exact_time,
91
96
  lat=box.data[0],
92
97
  lon=box.data[1],
93
98
  alt=box.data[2],
@@ -97,18 +102,40 @@ def _parse_point_from_sample(
97
102
  # Not using box.data.time_gps_epoch as the point timestamp
98
103
  # because it is from another clock
99
104
  return geo.Point(
100
- time=sample.time_offset,
105
+ time=sample.exact_time,
101
106
  lat=box.data.latitude,
102
107
  lon=box.data.longitude,
103
108
  alt=box.data.altitude,
104
109
  angle=None,
105
110
  )
111
+ elif box.type == CAMMType.ACCELERATION.value:
112
+ return telemetry.AccelerationData(
113
+ time=sample.exact_time,
114
+ x=box.data[0],
115
+ y=box.data[1],
116
+ z=box.data[2],
117
+ )
118
+ elif box.type == CAMMType.GYRO.value:
119
+ return telemetry.GyroscopeData(
120
+ time=sample.exact_time,
121
+ x=box.data[0],
122
+ y=box.data[1],
123
+ z=box.data[2],
124
+ )
125
+ elif box.type == CAMMType.MAGNETIC_FIELD.value:
126
+ return telemetry.MagnetometerData(
127
+ time=sample.exact_time,
128
+ x=box.data[0],
129
+ y=box.data[1],
130
+ z=box.data[2],
131
+ )
106
132
  return None
107
133
 
108
134
 
109
- def filter_points_by_elst(
110
- points: T.Iterable[geo.Point], elst: T.Sequence[T.Tuple[float, float]]
111
- ) -> T.Generator[geo.Point, None, None]:
135
+ def _filter_telemetry_by_elst_segments(
136
+ measurements: T.Iterable[TelemetryMeasurement],
137
+ elst: T.Sequence[T.Tuple[float, float]],
138
+ ) -> T.Generator[TelemetryMeasurement, None, None]:
112
139
  empty_elst = [entry for entry in elst if entry[0] == -1]
113
140
  if empty_elst:
114
141
  offset = empty_elst[-1][1]
@@ -118,20 +145,20 @@ def filter_points_by_elst(
118
145
  elst = [entry for entry in elst if entry[0] != -1]
119
146
 
120
147
  if not elst:
121
- for p in points:
122
- yield dataclasses.replace(p, time=p.time + offset)
148
+ for m in measurements:
149
+ yield dataclasses.replace(m, time=m.time + offset)
123
150
  return
124
151
 
125
152
  elst.sort(key=lambda entry: entry[0])
126
153
  elst_idx = 0
127
- for p in points:
154
+ for m in measurements:
128
155
  if len(elst) <= elst_idx:
129
156
  break
130
157
  media_time, duration = elst[elst_idx]
131
- if p.time < media_time:
158
+ if m.time < media_time:
132
159
  pass
133
- elif p.time <= media_time + duration:
134
- yield dataclasses.replace(p, time=p.time + offset)
160
+ elif m.time <= media_time + duration:
161
+ yield dataclasses.replace(m, time=m.time + offset)
135
162
  else:
136
163
  elst_idx += 1
137
164
 
@@ -148,15 +175,45 @@ def elst_entry_to_seconds(
148
175
  return (media_time, duration)
149
176
 
150
177
 
151
- def _extract_camm_samples(
152
- s: T.BinaryIO,
153
- maxsize: int = -1,
154
- ) -> T.Generator[sample_parser.Sample, None, None]:
155
- samples = sample_parser.parse_samples_from_trak(s, maxsize=maxsize)
156
- camm_samples = (
157
- sample for sample in samples if sample.description["format"] == b"camm"
158
- )
159
- yield from camm_samples
178
+ def _is_camm_description(description: T.Dict) -> bool:
179
+ return description["format"] == b"camm"
180
+
181
+
182
+ def _contains_camm_description(track: TrackBoxParser) -> bool:
183
+ descriptions = track.extract_sample_descriptions()
184
+ return any(_is_camm_description(d) for d in descriptions)
185
+
186
+
187
+ def _filter_telemetry_by_track_elst(
188
+ moov: MovieBoxParser,
189
+ track: TrackBoxParser,
190
+ measurements: T.Iterable[TelemetryMeasurement],
191
+ ) -> T.List[TelemetryMeasurement]:
192
+ elst_boxdata = track.extract_elst_boxdata()
193
+
194
+ if elst_boxdata is not None:
195
+ elst_entries = elst_boxdata["entries"]
196
+ if elst_entries:
197
+ # media_timescale
198
+ mdhd_boxdata = track.extract_mdhd_boxdata()
199
+ media_timescale = mdhd_boxdata["timescale"]
200
+
201
+ # movie_timescale
202
+ mvhd_boxdata = moov.extract_mvhd_boxdata()
203
+ movie_timescale = mvhd_boxdata["timescale"]
204
+
205
+ segments = [
206
+ elst_entry_to_seconds(
207
+ entry,
208
+ movie_timescale=movie_timescale,
209
+ media_timescale=media_timescale,
210
+ )
211
+ for entry in elst_entries
212
+ ]
213
+
214
+ return list(_filter_telemetry_by_elst_segments(measurements, segments))
215
+
216
+ return list(measurements)
160
217
 
161
218
 
162
219
  def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
@@ -165,62 +222,41 @@ def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
165
222
  otherwise None
166
223
  """
167
224
 
168
- points = None
169
- movie_timescale = None
170
- media_timescale = None
171
- elst_entries = None
225
+ moov = MovieBoxParser.parse_stream(fp)
172
226
 
173
- for h, s in parser.parse_path(fp, [b"moov", [b"mvhd", b"trak"]]):
174
- if h.type == b"trak":
175
- trak_start_offset = s.tell()
227
+ for track in moov.extract_tracks():
228
+ if _contains_camm_description(track):
229
+ maybe_measurements = (
230
+ _parse_telemetry_from_sample(fp, sample)
231
+ for sample in track.extract_samples()
232
+ if _is_camm_description(sample.description)
233
+ )
234
+ points = [m for m in maybe_measurements if isinstance(m, geo.Point)]
176
235
 
177
- descriptions = sample_parser.parse_descriptions_from_trak(
178
- s, maxsize=h.maxsize
236
+ return T.cast(
237
+ T.List[geo.Point], _filter_telemetry_by_track_elst(moov, track, points)
179
238
  )
180
- camm_descriptions = [d for d in descriptions if d["format"] == b"camm"]
181
- if camm_descriptions:
182
- s.seek(trak_start_offset, io.SEEK_SET)
183
- camm_samples = _extract_camm_samples(s, h.maxsize)
184
-
185
- points_with_nones = (
186
- _parse_point_from_sample(fp, sample)
187
- for sample in camm_samples
188
- if sample.description["format"] == b"camm"
189
- )
190
239
 
191
- points = [p for p in points_with_nones if p is not None]
192
- if points:
193
- s.seek(trak_start_offset)
194
- elst_data = parser.parse_box_data_first(
195
- s, [b"edts", b"elst"], maxsize=h.maxsize
196
- )
197
- if elst_data is not None:
198
- elst_entries = cparser.EditBox.parse(elst_data)["entries"]
199
-
200
- s.seek(trak_start_offset)
201
- mdhd_data = parser.parse_box_data_firstx(
202
- s, [b"mdia", b"mdhd"], maxsize=h.maxsize
203
- )
204
- mdhd = cparser.MediaHeaderBox.parse(mdhd_data)
205
- media_timescale = mdhd["timescale"]
206
- else:
207
- assert h.type == b"mvhd"
208
- if not movie_timescale:
209
- mvhd = cparser.MovieHeaderBox.parse(s.read(h.maxsize))
210
- movie_timescale = mvhd["timescale"]
240
+ return None
211
241
 
212
- # exit when both found
213
- if movie_timescale is not None and points:
214
- break
215
242
 
216
- if points and movie_timescale and media_timescale and elst_entries:
217
- segments = [
218
- elst_entry_to_seconds(entry, movie_timescale, media_timescale)
219
- for entry in elst_entries
220
- ]
221
- points = list(filter_points_by_elst(points, segments))
243
+ def extract_telemetry_data(fp: T.BinaryIO) -> T.Optional[T.List[TelemetryMeasurement]]:
244
+ moov = MovieBoxParser.parse_stream(fp)
222
245
 
223
- return points
246
+ for track in moov.extract_tracks():
247
+ if _contains_camm_description(track):
248
+ maybe_measurements = (
249
+ _parse_telemetry_from_sample(fp, sample)
250
+ for sample in track.extract_samples()
251
+ if _is_camm_description(sample.description)
252
+ )
253
+ measurements = [m for m in maybe_measurements if m is not None]
254
+
255
+ measurements = _filter_telemetry_by_track_elst(moov, track, measurements)
256
+
257
+ return measurements
258
+
259
+ return None
224
260
 
225
261
 
226
262
  def parse_gpx(path: pathlib.Path) -> T.List[geo.Point]:
@@ -238,7 +274,7 @@ MakeOrModel = C.Struct(
238
274
  )
239
275
 
240
276
 
241
- def _decode_quietly(data: bytes, h: parser.Header) -> str:
277
+ def _decode_quietly(data: bytes, h: sparser.Header) -> str:
242
278
  try:
243
279
  return data.decode("utf-8")
244
280
  except UnicodeDecodeError:
@@ -246,7 +282,7 @@ def _decode_quietly(data: bytes, h: parser.Header) -> str:
246
282
  return ""
247
283
 
248
284
 
249
- def _parse_quietly(data: bytes, h: parser.Header) -> bytes:
285
+ def _parse_quietly(data: bytes, h: sparser.Header) -> bytes:
250
286
  try:
251
287
  parsed = MakeOrModel.parse(data)
252
288
  except C.ConstructError:
@@ -256,7 +292,7 @@ def _parse_quietly(data: bytes, h: parser.Header) -> bytes:
256
292
 
257
293
 
258
294
  def extract_camera_make_and_model(fp: T.BinaryIO) -> T.Tuple[str, str]:
259
- header_and_stream = parser.parse_path(
295
+ header_and_stream = sparser.parse_path(
260
296
  fp,
261
297
  [
262
298
  b"moov",
@@ -296,7 +332,7 @@ def extract_camera_make_and_model(fp: T.BinaryIO) -> T.Tuple[str, str]:
296
332
  # quit when both found
297
333
  if make and model:
298
334
  break
299
- except parser.ParsingError:
335
+ except sparser.ParsingError:
300
336
  pass
301
337
 
302
338
  if make:
@@ -5,7 +5,6 @@ from . import (
5
5
  process_and_upload,
6
6
  sample_video,
7
7
  upload,
8
- upload_camm,
9
8
  video_process,
10
9
  video_process_and_upload,
11
10
  )
@@ -12,9 +12,6 @@ from . import (
12
12
  process_and_upload,
13
13
  sample_video,
14
14
  upload,
15
- upload_blackvue,
16
- upload_camm,
17
- upload_zip,
18
15
  video_process,
19
16
  video_process_and_upload,
20
17
  zip,
@@ -23,9 +20,6 @@ from . import (
23
20
  mapillary_tools_commands = [
24
21
  process,
25
22
  upload,
26
- upload_camm,
27
- upload_blackvue,
28
- upload_zip,
29
23
  sample_video,
30
24
  video_process,
31
25
  authenticate,
@@ -10,7 +10,6 @@ from ..process_geotag_properties import (
10
10
  process_finalize,
11
11
  process_geotag_properties,
12
12
  )
13
- from ..process_import_meta_properties import process_import_meta_properties
14
13
  from ..process_sequence_properties import process_sequence_properties
15
14
 
16
15
 
@@ -107,44 +106,6 @@ class Command:
107
106
  default=None,
108
107
  required=False,
109
108
  )
110
- group_metadata.add_argument(
111
- "--add_file_name",
112
- help="[DEPRECATED since v0.9.4] Add original file name to EXIF.",
113
- action="store_true",
114
- required=False,
115
- )
116
- group_metadata.add_argument(
117
- "--add_import_date",
118
- help="[DEPRECATED since v0.10.0] Add import date.",
119
- action="store_true",
120
- required=False,
121
- )
122
- group_metadata.add_argument(
123
- "--orientation",
124
- help="Specify the image orientation in degrees. Note this might result in image rotation. Note this input has precedence over the input read from the import source file.",
125
- choices=[0, 90, 180, 270],
126
- type=int,
127
- default=None,
128
- required=False,
129
- )
130
- group_metadata.add_argument(
131
- "--GPS_accuracy",
132
- help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.",
133
- default=None,
134
- required=False,
135
- )
136
- group_metadata.add_argument(
137
- "--camera_uuid",
138
- help="Custom string used to differentiate different captures taken with the same camera make and model.",
139
- default=None,
140
- required=False,
141
- )
142
- group_metadata.add_argument(
143
- "--custom_meta_data",
144
- help='[DEPRECATED since v0.10.0] Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234".',
145
- default=None,
146
- required=False,
147
- )
148
109
 
149
110
  group_geotagging = parser.add_argument_group(
150
111
  f"{constants.ANSI_BOLD}PROCESS GEOTAGGING OPTIONS{constants.ANSI_RESET_ALL}"
@@ -278,17 +239,6 @@ class Command:
278
239
  ),
279
240
  )
280
241
 
281
- metadatas = process_import_meta_properties(
282
- metadatas=metadatas,
283
- **(
284
- {
285
- k: v
286
- for k, v in vars_args.items()
287
- if k in inspect.getfullargspec(process_import_meta_properties).args
288
- }
289
- ),
290
- )
291
-
292
242
  metadatas = process_sequence_properties(
293
243
  metadatas=metadatas,
294
244
  **(
@@ -1,8 +1,7 @@
1
1
  import inspect
2
- import typing as T
3
2
 
4
3
  from .. import constants
5
- from ..upload import DirectUploadFileType, FileType, upload
4
+ from ..upload import upload
6
5
 
7
6
 
8
7
  class Command:
@@ -34,30 +33,6 @@ class Command:
34
33
  group = parser.add_argument_group(
35
34
  f"{constants.ANSI_BOLD}UPLOAD OPTIONS{constants.ANSI_RESET_ALL}"
36
35
  )
37
- default_filetypes = ",".join(sorted(t.value for t in FileType))
38
- supported_filetypes = ",".join(
39
- sorted(
40
- [t.value for t in DirectUploadFileType] + [t.value for t in FileType]
41
- )
42
- )
43
-
44
- def _type(option: str) -> T.List[T.Union[FileType, DirectUploadFileType]]:
45
- r: T.List[T.Union[FileType, DirectUploadFileType]] = []
46
- for t in option.split(","):
47
- if t in [x.value for x in FileType]:
48
- r.append(FileType(t))
49
- else:
50
- r.append(DirectUploadFileType(t))
51
- return r
52
-
53
- group.add_argument(
54
- "--filetypes",
55
- "--file_types",
56
- help=f"Upload files of the specified types only. Supported file types: {supported_filetypes} [default: %(default)s]",
57
- type=_type,
58
- default=default_filetypes,
59
- required=False,
60
- )
61
36
  group.add_argument(
62
37
  "--desc_path",
63
38
  help=f'Path to the description file generated by the process command. The hyphen "-" indicates STDIN. [default: {{IMPORT_PATH}}/{constants.IMAGE_DESCRIPTION_FILENAME}]',
@@ -43,8 +43,8 @@ GOPRO_GPS_PRECISION = float(os.getenv(_ENV_PREFIX + "GOPRO_GPS_PRECISION", 15))
43
43
 
44
44
  # WARNING: Changing the following envvars might result in failed uploads
45
45
  # Max number of images per sequence
46
- MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 500))
46
+ MAX_SEQUENCE_LENGTH = int(os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_LENGTH", 1000))
47
47
  # Max file size per sequence (sum of image filesizes in the sequence)
48
- MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "2G")
48
+ MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "10G")
49
49
  # Max number of pixels per sequence (sum of image pixels in the sequence)
50
50
  MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G")
@@ -5,6 +5,7 @@ import typing as T
5
5
  import xml.etree.ElementTree as ET
6
6
 
7
7
  from . import exif_read, exiftool_read, geo
8
+ from .telemetry import GPSFix, GPSPoint
8
9
 
9
10
 
10
11
  MAX_TRACK_ID = 10
@@ -87,7 +88,7 @@ def _aggregate_gps_track(
87
88
  alt_tag: T.Optional[str] = None,
88
89
  direction_tag: T.Optional[str] = None,
89
90
  ground_speed_tag: T.Optional[str] = None,
90
- ) -> T.List[geo.PointWithFix]:
91
+ ) -> T.List[GPSPoint]:
91
92
  """
92
93
  Aggregate all GPS data by the tags.
93
94
  It requires lat, lon to be present, and their lengths must match.
@@ -173,15 +174,16 @@ def _aggregate_gps_track(
173
174
  if timestamp is None or lon is None or lat is None:
174
175
  continue
175
176
  track.append(
176
- geo.PointWithFix(
177
+ GPSPoint(
177
178
  time=timestamp,
178
179
  lon=lon,
179
180
  lat=lat,
180
181
  alt=alt,
181
182
  angle=direction,
182
- gps_fix=None,
183
- gps_precision=None,
184
- gps_ground_speed=ground_speed,
183
+ epoch_time=None,
184
+ fix=None,
185
+ precision=None,
186
+ ground_speed=ground_speed,
185
187
  )
186
188
  )
187
189
 
@@ -230,8 +232,8 @@ def _aggregate_gps_track_by_sample_time(
230
232
  ground_speed_tag: T.Optional[str] = None,
231
233
  gps_fix_tag: T.Optional[str] = None,
232
234
  gps_precision_tag: T.Optional[str] = None,
233
- ) -> T.List[geo.PointWithFix]:
234
- track: T.List[geo.PointWithFix] = []
235
+ ) -> T.List[GPSPoint]:
236
+ track: T.List[GPSPoint] = []
235
237
 
236
238
  expanded_gps_fix_tag = None
237
239
  if gps_fix_tag is not None:
@@ -249,7 +251,7 @@ def _aggregate_gps_track_by_sample_time(
249
251
  gps_fix_texts = texts_by_tag.get(expanded_gps_fix_tag)
250
252
  if gps_fix_texts:
251
253
  try:
252
- gps_fix = geo.GPSFix(int(gps_fix_texts[0]))
254
+ gps_fix = GPSFix(int(gps_fix_texts[0]))
253
255
  except ValueError:
254
256
  gps_fix = None
255
257
 
@@ -280,7 +282,7 @@ def _aggregate_gps_track_by_sample_time(
280
282
  for idx, point in enumerate(points):
281
283
  point.time = sample_time + idx * avg_timedelta
282
284
  track.extend(
283
- dataclasses.replace(point, gps_fix=gps_fix, gps_precision=gps_precision)
285
+ dataclasses.replace(point, fix=gps_fix, precision=gps_precision)
284
286
  for point in points
285
287
  )
286
288
 
@@ -355,7 +357,7 @@ class ExifToolReadVideo:
355
357
  _, model = self._extract_make_and_model()
356
358
  return model
357
359
 
358
- def _extract_gps_track_from_track(self) -> T.List[geo.PointWithFix]:
360
+ def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
359
361
  for track_id in range(1, MAX_TRACK_ID + 1):
360
362
  track_ns = f"Track{track_id}"
361
363
  if self._all_tags_exists(
@@ -397,7 +399,7 @@ class ExifToolReadVideo:
397
399
 
398
400
  def _extract_gps_track_from_quicktime(
399
401
  self, namespace: str = "QuickTime"
400
- ) -> T.List[geo.PointWithFix]:
402
+ ) -> T.List[GPSPoint]:
401
403
  if not self._all_tags_exists(
402
404
  {
403
405
  expand_tag(f"{namespace}:GPSDateTime"),
mapillary_tools/ffmpeg.py CHANGED
@@ -202,10 +202,10 @@ class FFMPEG:
202
202
  return "0"
203
203
 
204
204
  if length == 1:
205
- return f"eq(n\\,{ sorted_frame_indices[0] })"
205
+ return f"eq(n\\,{sorted_frame_indices[0]})"
206
206
 
207
207
  middle = length // 2
208
- return f"if(lt(n\\,{ sorted_frame_indices[middle] })\\,{ self.generate_binary_search(sorted_frame_indices[:middle]) }\\,{ self.generate_binary_search(sorted_frame_indices[middle:]) })"
208
+ return f"if(lt(n\\,{sorted_frame_indices[middle]})\\,{self.generate_binary_search(sorted_frame_indices[:middle])}\\,{self.generate_binary_search(sorted_frame_indices[middle:])})"
209
209
 
210
210
  def extract_specified_frames(
211
211
  self,
mapillary_tools/geo.py CHANGED
@@ -6,7 +6,6 @@ import datetime
6
6
  import itertools
7
7
  import math
8
8
  import typing as T
9
- from enum import Enum, unique
10
9
 
11
10
  WGS84_a = 6378137.0
12
11
  WGS84_a_SQ = WGS84_a**2
@@ -32,45 +31,6 @@ class Point:
32
31
  angle: T.Optional[float]
33
32
 
34
33
 
35
- @unique
36
- class GPSFix(Enum):
37
- NO_FIX = 0
38
- FIX_2D = 2
39
- FIX_3D = 3
40
-
41
-
42
- @dataclasses.dataclass
43
- class PointWithFix(Point):
44
- gps_fix: T.Optional[GPSFix]
45
- gps_precision: T.Optional[float]
46
- gps_ground_speed: T.Optional[float]
47
-
48
-
49
- def _ecef_from_lla_DEPRECATED(
50
- lat: float, lon: float, alt: float
51
- ) -> T.Tuple[float, float, float]:
52
- """
53
- Deprecated because it is slow. Keep here for reference and comparison.
54
- Use _ecef_from_lla2 instead.
55
-
56
- Compute ECEF XYZ from latitude, longitude and altitude.
57
-
58
- All using the WGS94 model.
59
- Altitude is the distance to the WGS94 ellipsoid.
60
- Check results here http://www.oc.nps.edu/oc2902w/coord/llhxyz.htm
61
-
62
- """
63
- a2 = WGS84_a**2
64
- b2 = WGS84_b**2
65
- lat = math.radians(lat)
66
- lon = math.radians(lon)
67
- L = 1.0 / math.sqrt(a2 * math.cos(lat) ** 2 + b2 * math.sin(lat) ** 2)
68
- x = (a2 * L + alt) * math.cos(lat) * math.cos(lon)
69
- y = (a2 * L + alt) * math.cos(lat) * math.sin(lon)
70
- z = (b2 * L + alt) * math.sin(lat)
71
- return x, y, z
72
-
73
-
74
34
  def _ecef_from_lla2(lat: float, lon: float) -> T.Tuple[float, float, float]:
75
35
  """
76
36
  Compute ECEF XYZ from latitude, longitude and altitude.
@@ -172,20 +132,6 @@ def pairwise(iterable: T.Iterable[_IT]) -> T.Iterable[T.Tuple[_IT, _IT]]:
172
132
  return zip(a, b)
173
133
 
174
134
 
175
- def group_every(
176
- iterable: T.Iterable[_IT], n: int
177
- ) -> T.Generator[T.Generator[_IT, None, None], None, None]:
178
- """
179
- Return a generator that divides the iterable into groups by N.
180
- """
181
-
182
- if not (0 < n):
183
- raise ValueError("expect 0 < n but got {0}".format(n))
184
-
185
- for _, group in itertools.groupby(enumerate(iterable), key=lambda t: t[0] // n):
186
- yield (item for _, item in group)
187
-
188
-
189
135
  def as_unix_time(dt: T.Union[datetime.datetime, int, float]) -> float:
190
136
  if isinstance(dt, (int, float)):
191
137
  return dt
@@ -7,7 +7,7 @@ import typing as T
7
7
  import pynmea2
8
8
 
9
9
  from .. import geo
10
- from . import simple_mp4_parser
10
+ from ..mp4 import simple_mp4_parser as sparser
11
11
 
12
12
 
13
13
  LOG = logging.getLogger(__name__)
@@ -55,8 +55,8 @@ def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
55
55
 
56
56
  def extract_camera_model(fp: T.BinaryIO) -> str:
57
57
  try:
58
- cprt_bytes = simple_mp4_parser.parse_mp4_data_first(fp, [b"free", b"cprt"])
59
- except simple_mp4_parser.ParsingError:
58
+ cprt_bytes = sparser.parse_mp4_data_first(fp, [b"free", b"cprt"])
59
+ except sparser.ParsingError:
60
60
  return ""
61
61
 
62
62
  if cprt_bytes is None:
@@ -91,7 +91,7 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
91
91
 
92
92
 
93
93
  def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
94
- gps_data = simple_mp4_parser.parse_mp4_data_first(fp, [b"free", b"gps "])
94
+ gps_data = sparser.parse_mp4_data_first(fp, [b"free", b"gps "])
95
95
  if gps_data is None:
96
96
  return None
97
97
 
@@ -6,7 +6,7 @@ from pathlib import Path
6
6
 
7
7
  from tqdm import tqdm
8
8
 
9
- from .. import exceptions, exif_write, geo, types
9
+ from .. import exceptions, exif_write, geo, types, utils
10
10
  from ..exif_read import ExifRead, ExifReadABC
11
11
  from .geotag_from_generic import GeotagImagesFromGeneric
12
12
 
@@ -64,6 +64,7 @@ class GeotagImagesFromEXIF(GeotagImagesFromGeneric):
64
64
  image_metadata = types.ImageMetadata(
65
65
  filename=image_path,
66
66
  md5sum=None,
67
+ filesize=utils.get_file_size(image_path),
67
68
  time=geo.as_unix_time(capture_time),
68
69
  lat=lat,
69
70
  lon=lon,
@@ -48,7 +48,6 @@ class GeotagImagesFromExifToolBothImageAndVideo(GeotagImagesFromGeneric):
48
48
  video_paths = utils.find_videos(
49
49
  [Path(pathstr) for pathstr in rdf_description_by_path.keys()],
50
50
  skip_subfolders=True,
51
- check_file_suffix=True,
52
51
  )
53
52
 
54
53
  # will try to geotag these error metadatas from video later