mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. mapillary_tools/__init__.py +1 -1
  2. mapillary_tools/api_v4.py +237 -16
  3. mapillary_tools/authenticate.py +325 -64
  4. mapillary_tools/{geotag/blackvue_parser.py → blackvue_parser.py} +74 -54
  5. mapillary_tools/camm/camm_builder.py +55 -97
  6. mapillary_tools/camm/camm_parser.py +429 -181
  7. mapillary_tools/commands/__main__.py +12 -6
  8. mapillary_tools/commands/authenticate.py +8 -1
  9. mapillary_tools/commands/process.py +27 -51
  10. mapillary_tools/commands/process_and_upload.py +19 -5
  11. mapillary_tools/commands/sample_video.py +2 -3
  12. mapillary_tools/commands/upload.py +18 -9
  13. mapillary_tools/commands/video_process_and_upload.py +19 -5
  14. mapillary_tools/config.py +31 -13
  15. mapillary_tools/constants.py +47 -6
  16. mapillary_tools/exceptions.py +34 -35
  17. mapillary_tools/exif_read.py +221 -116
  18. mapillary_tools/exif_write.py +7 -7
  19. mapillary_tools/exiftool_read.py +33 -42
  20. mapillary_tools/exiftool_read_video.py +46 -33
  21. mapillary_tools/exiftool_runner.py +77 -0
  22. mapillary_tools/ffmpeg.py +24 -23
  23. mapillary_tools/geo.py +144 -120
  24. mapillary_tools/geotag/base.py +147 -0
  25. mapillary_tools/geotag/factory.py +291 -0
  26. mapillary_tools/geotag/geotag_images_from_exif.py +14 -131
  27. mapillary_tools/geotag/geotag_images_from_exiftool.py +126 -82
  28. mapillary_tools/geotag/geotag_images_from_gpx.py +53 -118
  29. mapillary_tools/geotag/geotag_images_from_gpx_file.py +13 -126
  30. mapillary_tools/geotag/geotag_images_from_nmea_file.py +4 -5
  31. mapillary_tools/geotag/geotag_images_from_video.py +53 -51
  32. mapillary_tools/geotag/geotag_videos_from_exiftool.py +97 -0
  33. mapillary_tools/geotag/geotag_videos_from_gpx.py +39 -0
  34. mapillary_tools/geotag/geotag_videos_from_video.py +20 -185
  35. mapillary_tools/geotag/image_extractors/base.py +18 -0
  36. mapillary_tools/geotag/image_extractors/exif.py +60 -0
  37. mapillary_tools/geotag/image_extractors/exiftool.py +18 -0
  38. mapillary_tools/geotag/options.py +160 -0
  39. mapillary_tools/geotag/utils.py +52 -16
  40. mapillary_tools/geotag/video_extractors/base.py +18 -0
  41. mapillary_tools/geotag/video_extractors/exiftool.py +70 -0
  42. mapillary_tools/{video_data_extraction/extractors/gpx_parser.py → geotag/video_extractors/gpx.py} +57 -39
  43. mapillary_tools/geotag/video_extractors/native.py +157 -0
  44. mapillary_tools/{geotag → gpmf}/gpmf_parser.py +205 -182
  45. mapillary_tools/{geotag → gpmf}/gps_filter.py +5 -3
  46. mapillary_tools/history.py +7 -13
  47. mapillary_tools/mp4/construct_mp4_parser.py +9 -8
  48. mapillary_tools/mp4/io_utils.py +0 -1
  49. mapillary_tools/mp4/mp4_sample_parser.py +36 -28
  50. mapillary_tools/mp4/simple_mp4_builder.py +10 -9
  51. mapillary_tools/mp4/simple_mp4_parser.py +13 -22
  52. mapillary_tools/process_geotag_properties.py +155 -392
  53. mapillary_tools/process_sequence_properties.py +562 -208
  54. mapillary_tools/sample_video.py +13 -20
  55. mapillary_tools/telemetry.py +26 -13
  56. mapillary_tools/types.py +111 -58
  57. mapillary_tools/upload.py +316 -298
  58. mapillary_tools/upload_api_v4.py +55 -122
  59. mapillary_tools/uploader.py +396 -254
  60. mapillary_tools/utils.py +42 -18
  61. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/METADATA +3 -2
  62. mapillary_tools-0.14.0a2.dist-info/RECORD +72 -0
  63. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/WHEEL +1 -1
  64. mapillary_tools/geotag/__init__.py +0 -1
  65. mapillary_tools/geotag/geotag_from_generic.py +0 -22
  66. mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py +0 -93
  67. mapillary_tools/geotag/geotag_videos_from_exiftool_video.py +0 -145
  68. mapillary_tools/video_data_extraction/cli_options.py +0 -22
  69. mapillary_tools/video_data_extraction/extract_video_data.py +0 -176
  70. mapillary_tools/video_data_extraction/extractors/base_parser.py +0 -75
  71. mapillary_tools/video_data_extraction/extractors/blackvue_parser.py +0 -34
  72. mapillary_tools/video_data_extraction/extractors/camm_parser.py +0 -38
  73. mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py +0 -71
  74. mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py +0 -53
  75. mapillary_tools/video_data_extraction/extractors/generic_video_parser.py +0 -52
  76. mapillary_tools/video_data_extraction/extractors/gopro_parser.py +0 -43
  77. mapillary_tools/video_data_extraction/extractors/nmea_parser.py +0 -24
  78. mapillary_tools/video_data_extraction/video_data_parser_factory.py +0 -39
  79. mapillary_tools-0.13.3a1.dist-info/RECORD +0 -75
  80. /mapillary_tools/{geotag → gpmf}/gpmf_gps_filter.py +0 -0
  81. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/entry_points.txt +0 -0
  82. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info/licenses}/LICENSE +0 -0
  83. {mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/top_level.txt +0 -0
@@ -1,109 +1,153 @@
1
- import io
1
+ from __future__ import annotations
2
+
2
3
  import logging
4
+ import sys
3
5
  import typing as T
4
6
  import xml.etree.ElementTree as ET
5
- from multiprocessing import Pool
6
7
  from pathlib import Path
7
8
 
8
- from tqdm import tqdm
9
+ if sys.version_info >= (3, 12):
10
+ from typing import override
11
+ else:
12
+ from typing_extensions import override
9
13
 
10
- from .. import exceptions, exiftool_read, types
11
- from .geotag_from_generic import GeotagImagesFromGeneric
12
- from .geotag_images_from_exif import GeotagImagesFromEXIF, verify_image_exif_write
14
+ from .. import constants, exceptions, exiftool_read, types, utils
15
+ from ..exiftool_runner import ExiftoolRunner
16
+ from .base import GeotagImagesFromGeneric
17
+ from .geotag_images_from_video import GeotagImagesFromVideo
18
+ from .geotag_videos_from_exiftool import GeotagVideosFromExifToolXML
19
+ from .image_extractors.exiftool import ImageExifToolExtractor
20
+ from .utils import index_rdf_description_by_path
13
21
 
14
22
  LOG = logging.getLogger(__name__)
15
23
 
16
24
 
17
- class GeotagImagesFromExifTool(GeotagImagesFromGeneric):
25
+ class GeotagImagesFromExifToolXML(GeotagImagesFromGeneric):
18
26
  def __init__(
19
27
  self,
20
- image_paths: T.Sequence[Path],
21
28
  xml_path: Path,
22
- num_processes: T.Optional[int] = None,
29
+ num_processes: int | None = None,
23
30
  ):
24
- self.image_paths = image_paths
25
31
  self.xml_path = xml_path
26
- self.num_processes = num_processes
27
- super().__init__()
32
+ super().__init__(num_processes=num_processes)
33
+
34
+ @classmethod
35
+ def build_image_extractors(
36
+ cls,
37
+ rdf_by_path: dict[str, ET.Element],
38
+ image_paths: T.Iterable[Path],
39
+ ) -> list[ImageExifToolExtractor | types.ErrorMetadata]:
40
+ results: list[ImageExifToolExtractor | types.ErrorMetadata] = []
41
+
42
+ for path in image_paths:
43
+ rdf = rdf_by_path.get(exiftool_read.canonical_path(path))
44
+ if rdf is None:
45
+ ex = exceptions.MapillaryExifToolXMLNotFoundError(
46
+ "Cannot find the image in the ExifTool XML"
47
+ )
48
+ results.append(
49
+ types.describe_error_metadata(
50
+ ex, path, filetype=types.FileType.IMAGE
51
+ )
52
+ )
53
+ else:
54
+ results.append(ImageExifToolExtractor(path, rdf))
55
+
56
+ return results
57
+
58
+ @override
59
+ def _generate_image_extractors(
60
+ self, image_paths: T.Sequence[Path]
61
+ ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
62
+ rdf_by_path = index_rdf_description_by_path([self.xml_path])
63
+ return self.build_image_extractors(rdf_by_path, image_paths)
28
64
 
29
- @staticmethod
30
- def geotag_image(element: ET.Element) -> types.ImageMetadataOrError:
31
- image_path = exiftool_read.find_rdf_description_path(element)
32
- assert image_path is not None, "must find the path from the element"
33
65
 
66
+ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
67
+ @override
68
+ def _generate_image_extractors(
69
+ self, image_paths: T.Sequence[Path]
70
+ ) -> T.Sequence[ImageExifToolExtractor | types.ErrorMetadata]:
71
+ runner = ExiftoolRunner(constants.EXIFTOOL_PATH)
72
+
73
+ LOG.debug(
74
+ "Extracting XML from %d images with ExifTool command: %s",
75
+ len(image_paths),
76
+ " ".join(runner._build_args_read_stdin()),
77
+ )
34
78
  try:
35
- exif = exiftool_read.ExifToolRead(ET.ElementTree(element))
36
- image_metadata = GeotagImagesFromEXIF.build_image_metadata(
37
- image_path, exif, skip_lonlat_error=False
38
- )
39
- # load the image bytes into memory to avoid reading it multiple times
40
- with image_path.open("rb") as fp:
41
- image_bytesio = io.BytesIO(fp.read())
42
- image_bytesio.seek(0, io.SEEK_SET)
43
- verify_image_exif_write(
44
- image_metadata,
45
- image_bytes=image_bytesio.read(),
79
+ xml = runner.extract_xml(image_paths)
80
+ except FileNotFoundError as ex:
81
+ raise exceptions.MapillaryExiftoolNotFoundError(ex) from ex
82
+
83
+ try:
84
+ xml_element = ET.fromstring(xml)
85
+ except ET.ParseError as ex:
86
+ LOG.warning(
87
+ "Failed to parse ExifTool XML: %s",
88
+ str(ex),
89
+ exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
46
90
  )
47
- except Exception as ex:
48
- return types.describe_error_metadata(
49
- ex, image_path, filetype=types.FileType.IMAGE
91
+ rdf_by_path = {}
92
+ else:
93
+ rdf_by_path = exiftool_read.index_rdf_description_by_path_from_xml_element(
94
+ xml_element
50
95
  )
51
96
 
52
- image_bytesio.seek(0, io.SEEK_SET)
53
- image_metadata.update_md5sum(image_bytesio)
97
+ return GeotagImagesFromExifToolXML.build_image_extractors(
98
+ rdf_by_path, image_paths
99
+ )
54
100
 
55
- return image_metadata
56
101
 
57
- def to_description(self) -> T.List[types.ImageMetadataOrError]:
58
- rdf_description_by_path = exiftool_read.index_rdf_description_by_path(
59
- [self.xml_path]
102
+ class GeotagImagesFromExifToolWithSamples(GeotagImagesFromGeneric):
103
+ def __init__(
104
+ self,
105
+ xml_path: Path,
106
+ offset_time: float = 0.0,
107
+ num_processes: int | None = None,
108
+ ):
109
+ super().__init__(num_processes=num_processes)
110
+ self.xml_path = xml_path
111
+ self.offset_time = offset_time
112
+
113
+ def geotag_samples(
114
+ self, image_paths: T.Sequence[Path]
115
+ ) -> list[types.ImageMetadataOrError]:
116
+ # Find all video paths in self.xml_path
117
+ rdf_by_path = index_rdf_description_by_path([self.xml_path])
118
+ video_paths = utils.find_videos(
119
+ [Path(pathstr) for pathstr in rdf_by_path.keys()],
120
+ skip_subfolders=True,
60
121
  )
122
+ # Find all video paths that have sample images
123
+ samples_by_video = utils.find_all_image_samples(image_paths, video_paths)
61
124
 
62
- error_metadatas: T.List[types.ErrorMetadata] = []
63
- rdf_descriptions: T.List[ET.Element] = []
64
- for path in self.image_paths:
65
- rdf_description = rdf_description_by_path.get(
66
- exiftool_read.canonical_path(path)
67
- )
68
- if rdf_description is None:
69
- exc = exceptions.MapillaryEXIFNotFoundError(
70
- f"The {exiftool_read._DESCRIPTION_TAG} XML element for the image not found"
71
- )
72
- error_metadatas.append(
73
- types.describe_error_metadata(
74
- exc, path, filetype=types.FileType.IMAGE
75
- )
76
- )
77
- else:
78
- rdf_descriptions.append(rdf_description)
125
+ video_metadata_or_errors = GeotagVideosFromExifToolXML(
126
+ self.xml_path,
127
+ num_processes=self.num_processes,
128
+ ).to_description(list(samples_by_video.keys()))
129
+ sample_paths = sum(samples_by_video.values(), [])
130
+ sample_metadata_or_errors = GeotagImagesFromVideo(
131
+ video_metadata_or_errors,
132
+ offset_time=self.offset_time,
133
+ num_processes=self.num_processes,
134
+ ).to_description(sample_paths)
79
135
 
80
- if self.num_processes is None:
81
- num_processes = self.num_processes
82
- disable_multiprocessing = False
83
- else:
84
- num_processes = max(self.num_processes, 1)
85
- disable_multiprocessing = self.num_processes <= 0
86
-
87
- with Pool(processes=num_processes) as pool:
88
- image_metadatas_iter: T.Iterator[types.ImageMetadataOrError]
89
- if disable_multiprocessing:
90
- image_metadatas_iter = map(
91
- GeotagImagesFromExifTool.geotag_image,
92
- rdf_descriptions,
93
- )
94
- else:
95
- image_metadatas_iter = pool.imap(
96
- GeotagImagesFromExifTool.geotag_image,
97
- rdf_descriptions,
98
- )
99
- image_metadata_or_errors = list(
100
- tqdm(
101
- image_metadatas_iter,
102
- desc="Extracting geotags from ExifTool XML",
103
- unit="images",
104
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
105
- total=len(self.image_paths),
106
- )
107
- )
136
+ return sample_metadata_or_errors
137
+
138
+ @override
139
+ def to_description(
140
+ self, image_paths: T.Sequence[Path]
141
+ ) -> list[types.ImageMetadataOrError]:
142
+ sample_metadata_or_errors = self.geotag_samples(image_paths)
143
+
144
+ sample_paths = set(metadata.filename for metadata in sample_metadata_or_errors)
145
+
146
+ non_sample_paths = [path for path in image_paths if path not in sample_paths]
147
+
148
+ non_sample_metadata_or_errors = GeotagImagesFromExifToolXML(
149
+ self.xml_path,
150
+ num_processes=self.num_processes,
151
+ ).to_description(non_sample_paths)
108
152
 
109
- return error_metadatas + image_metadata_or_errors
153
+ return sample_metadata_or_errors + non_sample_metadata_or_errors
@@ -1,12 +1,19 @@
1
+ from __future__ import annotations
2
+
1
3
  import dataclasses
2
4
  import logging
5
+ import sys
3
6
  import typing as T
4
- from multiprocessing import Pool
5
7
  from pathlib import Path
6
8
 
9
+ if sys.version_info >= (3, 12):
10
+ from typing import override
11
+ else:
12
+ from typing_extensions import override
13
+
7
14
  from .. import exceptions, geo, types
8
- from .geotag_from_generic import GeotagImagesFromGeneric
9
- from .geotag_images_from_exif import GeotagImagesFromEXIF
15
+ from .base import GeotagImagesFromGeneric
16
+ from .geotag_images_from_exif import ImageEXIFExtractor
10
17
 
11
18
 
12
19
  LOG = logging.getLogger(__name__)
@@ -15,46 +22,23 @@ LOG = logging.getLogger(__name__)
15
22
  class GeotagImagesFromGPX(GeotagImagesFromGeneric):
16
23
  def __init__(
17
24
  self,
18
- image_paths: T.Sequence[Path],
19
25
  points: T.Sequence[geo.Point],
20
26
  use_gpx_start_time: bool = False,
21
27
  use_image_start_time: bool = False,
22
28
  offset_time: float = 0.0,
23
- num_processes: T.Optional[int] = None,
29
+ num_processes: int | None = None,
24
30
  ):
25
- super().__init__()
26
- self.image_paths = image_paths
31
+ super().__init__(num_processes=num_processes)
27
32
  self.points = points
28
33
  self.use_gpx_start_time = use_gpx_start_time
29
34
  self.use_image_start_time = use_image_start_time
30
35
  self.offset_time = offset_time
31
- self.num_processes = num_processes
32
-
33
- @staticmethod
34
- def geotag_image(image_path: Path) -> types.ImageMetadataOrError:
35
- return GeotagImagesFromEXIF.geotag_image(image_path, skip_lonlat_error=True)
36
-
37
- def geotag_multiple_images(
38
- self, image_paths: T.Sequence[Path]
39
- ) -> T.List[types.ImageMetadataOrError]:
40
- if self.num_processes is None:
41
- num_processes = self.num_processes
42
- disable_multiprocessing = False
43
- else:
44
- num_processes = max(self.num_processes, 1)
45
- disable_multiprocessing = self.num_processes <= 0
46
-
47
- if disable_multiprocessing:
48
- return list(map(GeotagImagesFromGPX.geotag_image, image_paths))
49
- else:
50
- with Pool(processes=num_processes) as pool:
51
- return pool.map(GeotagImagesFromGPX.geotag_image, image_paths)
52
36
 
53
37
  def _interpolate_image_metadata_along(
54
38
  self,
55
39
  image_metadata: types.ImageMetadata,
56
40
  sorted_points: T.Sequence[geo.Point],
57
- ) -> types.ImageMetadataOrError:
41
+ ) -> types.ImageMetadata:
58
42
  assert sorted_points, "must have at least one point"
59
43
 
60
44
  if image_metadata.time < sorted_points[0].time:
@@ -63,15 +47,12 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
63
47
  gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
64
48
  # with the tolerance of 1ms
65
49
  if 0.001 < delta:
66
- exc = exceptions.MapillaryOutsideGPXTrackError(
50
+ raise exceptions.MapillaryOutsideGPXTrackError(
67
51
  f"The image date time is {round(delta, 3)} seconds behind the GPX start point",
68
52
  image_time=types.datetime_to_map_capture_time(image_metadata.time),
69
53
  gpx_start_time=gpx_start_time,
70
54
  gpx_end_time=gpx_end_time,
71
55
  )
72
- return types.describe_error_metadata(
73
- exc, image_metadata.filename, filetype=types.FileType.IMAGE
74
- )
75
56
 
76
57
  if sorted_points[-1].time < image_metadata.time:
77
58
  delta = image_metadata.time - sorted_points[-1].time
@@ -79,15 +60,12 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
79
60
  gpx_end_time = types.datetime_to_map_capture_time(sorted_points[-1].time)
80
61
  # with the tolerance of 1ms
81
62
  if 0.001 < delta:
82
- exc = exceptions.MapillaryOutsideGPXTrackError(
63
+ raise exceptions.MapillaryOutsideGPXTrackError(
83
64
  f"The image time is {round(delta, 3)} seconds beyond the GPX end point",
84
65
  image_time=types.datetime_to_map_capture_time(image_metadata.time),
85
66
  gpx_start_time=gpx_start_time,
86
67
  gpx_end_time=gpx_end_time,
87
68
  )
88
- return types.describe_error_metadata(
89
- exc, image_metadata.filename, filetype=types.FileType.IMAGE
90
- )
91
69
 
92
70
  interpolated = geo.interpolate(sorted_points, image_metadata.time)
93
71
 
@@ -100,34 +78,30 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
100
78
  time=interpolated.time,
101
79
  )
102
80
 
103
- def to_description(self) -> T.List[types.ImageMetadataOrError]:
104
- metadatas: T.List[types.ImageMetadataOrError] = []
105
-
106
- if not self.points:
107
- exc = exceptions.MapillaryGPXEmptyError(
108
- "Empty GPS extracted from the geotag source"
109
- )
110
- for image_path in self.image_paths:
111
- metadatas.append(
112
- types.describe_error_metadata(
113
- exc, image_path, filetype=types.FileType.IMAGE
114
- ),
115
- )
116
- assert len(self.image_paths) == len(metadatas)
117
- return metadatas
81
+ @override
82
+ def _generate_image_extractors(
83
+ self, image_paths: T.Sequence[Path]
84
+ ) -> T.Sequence[ImageEXIFExtractor]:
85
+ return [
86
+ ImageEXIFExtractor(path, skip_lonlat_error=True) for path in image_paths
87
+ ]
88
+
89
+ @override
90
+ def to_description(
91
+ self, image_paths: T.Sequence[Path]
92
+ ) -> list[types.ImageMetadataOrError]:
93
+ final_metadatas: list[types.ImageMetadataOrError] = []
118
94
 
119
- image_metadata_or_errors = self.geotag_multiple_images(self.image_paths)
95
+ image_metadata_or_errors = super().to_description(image_paths)
120
96
 
121
- image_metadatas = []
122
- for metadata_or_error in image_metadata_or_errors:
123
- if isinstance(metadata_or_error, types.ErrorMetadata):
124
- metadatas.append(metadata_or_error)
125
- else:
126
- image_metadatas.append(metadata_or_error)
97
+ image_metadatas, error_metadatas = types.separate_errors(
98
+ image_metadata_or_errors
99
+ )
100
+ final_metadatas.extend(error_metadatas)
127
101
 
128
102
  if not image_metadatas:
129
- assert len(self.image_paths) == len(metadatas)
130
- return metadatas
103
+ assert len(image_paths) == len(final_metadatas)
104
+ return final_metadatas
131
105
 
132
106
  # Do not use point itself for comparison because point.angle or point.alt could be None
133
107
  # when you compare nonnull value with None, it will throw
@@ -162,64 +136,25 @@ class GeotagImagesFromGPX(GeotagImagesFromGeneric):
162
136
  LOG.debug("GPX start time delta: %s", time_delta)
163
137
  image_time_offset += time_delta
164
138
 
165
- LOG.debug("Final time offset for interpolation: %s", image_time_offset)
139
+ if image_time_offset:
140
+ LOG.debug("Final time offset for interpolation: %s", image_time_offset)
141
+ for image_metadata in sorted_image_metadatas:
142
+ # TODO: this time modification seems to affect final capture times
143
+ image_metadata.time += image_time_offset
166
144
 
167
145
  for image_metadata in sorted_image_metadatas:
168
- image_metadata.time += image_time_offset
169
- metadatas.append(
170
- self._interpolate_image_metadata_along(image_metadata, sorted_points)
171
- )
172
-
173
- assert len(self.image_paths) == len(metadatas)
174
- return metadatas
175
-
176
-
177
- class GeotagImagesFromGPXWithProgress(GeotagImagesFromGPX):
178
- def __init__(
179
- self,
180
- image_paths: T.Sequence[Path],
181
- points: T.Sequence[geo.Point],
182
- use_gpx_start_time: bool = False,
183
- use_image_start_time: bool = False,
184
- offset_time: float = 0.0,
185
- num_processes: T.Optional[int] = None,
186
- progress_bar=None,
187
- ) -> None:
188
- super().__init__(
189
- image_paths,
190
- points,
191
- use_gpx_start_time=use_gpx_start_time,
192
- use_image_start_time=use_image_start_time,
193
- offset_time=offset_time,
194
- num_processes=num_processes,
195
- )
196
- self._progress_bar = progress_bar
197
-
198
- def geotag_multiple_images(
199
- self, image_paths: T.Sequence[Path]
200
- ) -> T.List[types.ImageMetadataOrError]:
201
- if self._progress_bar is None:
202
- return super().geotag_multiple_images(image_paths)
203
-
204
- if self.num_processes is None:
205
- num_processes = self.num_processes
206
- disable_multiprocessing = False
207
- else:
208
- num_processes = max(self.num_processes, 1)
209
- disable_multiprocessing = self.num_processes <= 0
210
-
211
- output = []
212
- with Pool(processes=num_processes) as pool:
213
- image_metadatas_iter: T.Iterator[types.ImageMetadataOrError]
214
- if disable_multiprocessing:
215
- image_metadatas_iter = map(
216
- GeotagImagesFromGPX.geotag_image, image_paths
146
+ try:
147
+ final_metadatas.append(
148
+ self._interpolate_image_metadata_along(
149
+ image_metadata, sorted_points
150
+ )
217
151
  )
218
- else:
219
- image_metadatas_iter = pool.imap(
220
- GeotagImagesFromGPX.geotag_image, image_paths
152
+ except exceptions.MapillaryOutsideGPXTrackError as ex:
153
+ error_metadata = types.describe_error_metadata(
154
+ ex, image_metadata.filename, filetype=types.FileType.IMAGE
221
155
  )
222
- for image_metadata_or_error in image_metadatas_iter:
223
- self._progress_bar.update(1)
224
- output.append(image_metadata_or_error)
225
- return output
156
+ final_metadatas.append(error_metadata)
157
+
158
+ assert len(image_paths) == len(final_metadatas)
159
+
160
+ return final_metadatas
@@ -1,32 +1,25 @@
1
- import dataclasses
1
+ from __future__ import annotations
2
+
2
3
  import logging
3
- import typing as T
4
- from multiprocessing import Pool
5
4
  from pathlib import Path
6
5
 
7
- import gpxpy
8
- from tqdm import tqdm
9
-
10
- from .. import exif_read, geo, types
11
- from .geotag_from_generic import GeotagImagesFromGeneric
12
- from .geotag_images_from_gpx import GeotagImagesFromGPXWithProgress
6
+ from . import utils
7
+ from .geotag_images_from_gpx import GeotagImagesFromGPX
13
8
 
14
9
 
15
10
  LOG = logging.getLogger(__name__)
16
11
 
17
12
 
18
- class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
13
+ class GeotagImagesFromGPXFile(GeotagImagesFromGPX):
19
14
  def __init__(
20
15
  self,
21
- image_paths: T.Sequence[Path],
22
16
  source_path: Path,
23
17
  use_gpx_start_time: bool = False,
24
18
  offset_time: float = 0.0,
25
- num_processes: T.Optional[int] = None,
19
+ num_processes: int | None = None,
26
20
  ):
27
- super().__init__()
28
21
  try:
29
- tracks = parse_gpx(source_path)
22
+ tracks = utils.parse_gpx(source_path)
30
23
  except Exception as ex:
31
24
  raise RuntimeError(
32
25
  f"Error parsing GPX {source_path}: {ex.__class__.__name__}: {ex}"
@@ -38,116 +31,10 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
38
31
  len(tracks),
39
32
  source_path,
40
33
  )
41
- self.points: T.List[geo.Point] = sum(tracks, [])
42
- self.image_paths = image_paths
43
- self.source_path = source_path
44
- self.use_gpx_start_time = use_gpx_start_time
45
- self.offset_time = offset_time
46
- self.num_processes = num_processes
47
-
48
- @staticmethod
49
- def _extract_image_metadata(
50
- image_metadata: types.ImageMetadata,
51
- ) -> types.ImageMetadataOrError:
52
- try:
53
- exif = exif_read.ExifRead(image_metadata.filename)
54
- orientation = exif.extract_orientation()
55
- make = exif.extract_make()
56
- model = exif.extract_model()
57
- except Exception as ex:
58
- return types.describe_error_metadata(
59
- ex, image_metadata.filename, filetype=types.FileType.IMAGE
60
- )
61
-
62
- return dataclasses.replace(
63
- image_metadata,
64
- MAPOrientation=orientation,
65
- MAPDeviceMake=make,
66
- MAPDeviceModel=model,
67
- )
68
-
69
- def to_description(self) -> T.List[types.ImageMetadataOrError]:
70
- with tqdm(
71
- total=len(self.image_paths),
72
- desc="Interpolating",
73
- unit="images",
74
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
75
- ) as pbar:
76
- geotag = GeotagImagesFromGPXWithProgress(
77
- self.image_paths,
78
- self.points,
79
- use_gpx_start_time=self.use_gpx_start_time,
80
- offset_time=self.offset_time,
81
- progress_bar=pbar,
82
- )
83
- image_metadata_or_errors = geotag.to_description()
84
-
85
- image_metadatas: T.List[types.ImageMetadata] = []
86
- error_metadatas: T.List[types.ErrorMetadata] = []
87
- for metadata in image_metadata_or_errors:
88
- if isinstance(metadata, types.ErrorMetadata):
89
- error_metadatas.append(metadata)
90
- else:
91
- image_metadatas.append(metadata)
92
-
93
- if self.num_processes is None:
94
- num_processes = self.num_processes
95
- disable_multiprocessing = False
96
- else:
97
- num_processes = max(self.num_processes, 1)
98
- disable_multiprocessing = self.num_processes <= 0
99
-
100
- with Pool(processes=num_processes) as pool:
101
- image_metadatas_iter: T.Iterator[types.ImageMetadataOrError]
102
- if disable_multiprocessing:
103
- image_metadatas_iter = map(
104
- GeotagImagesFromGPXFile._extract_image_metadata, image_metadatas
105
- )
106
- else:
107
- # Do not pass error metadatas where the error object can not be pickled for multiprocessing to work
108
- # Otherwise we get:
109
- # TypeError: __init__() missing 3 required positional arguments: 'image_time', 'gpx_start_time', and 'gpx_end_time'
110
- # See https://stackoverflow.com/a/61432070
111
- image_metadatas_iter = pool.imap(
112
- GeotagImagesFromGPXFile._extract_image_metadata, image_metadatas
113
- )
114
- image_metadata_or_errors = list(
115
- tqdm(
116
- image_metadatas_iter,
117
- desc="Processing",
118
- unit="images",
119
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
120
- )
121
- )
122
-
123
- return (
124
- T.cast(T.List[types.ImageMetadataOrError], error_metadatas)
125
- + image_metadata_or_errors
34
+ points = sum(tracks, [])
35
+ super().__init__(
36
+ points,
37
+ use_gpx_start_time=use_gpx_start_time,
38
+ offset_time=offset_time,
39
+ num_processes=num_processes,
126
40
  )
127
-
128
-
129
- Track = T.List[geo.Point]
130
-
131
-
132
- def parse_gpx(gpx_file: Path) -> T.List[Track]:
133
- with gpx_file.open("r") as f:
134
- gpx = gpxpy.parse(f)
135
-
136
- tracks: T.List[Track] = []
137
-
138
- for track in gpx.tracks:
139
- for segment in track.segments:
140
- tracks.append([])
141
- for point in segment.points:
142
- if point.time is not None:
143
- tracks[-1].append(
144
- geo.Point(
145
- time=geo.as_unix_time(point.time),
146
- lat=point.latitude,
147
- lon=point.longitude,
148
- alt=point.elevation,
149
- angle=None,
150
- )
151
- )
152
-
153
- return tracks
@@ -1,5 +1,6 @@
1
+ from __future__ import annotations
2
+
1
3
  import datetime
2
- import typing as T
3
4
  from pathlib import Path
4
5
 
5
6
  import pynmea2
@@ -11,15 +12,13 @@ from .geotag_images_from_gpx import GeotagImagesFromGPX
11
12
  class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
12
13
  def __init__(
13
14
  self,
14
- image_paths: T.Sequence[Path],
15
15
  source_path: Path,
16
16
  use_gpx_start_time: bool = False,
17
17
  offset_time: float = 0.0,
18
- num_processes: T.Optional[int] = None,
18
+ num_processes: int | None = None,
19
19
  ):
20
20
  points = get_lat_lon_time_from_nmea(source_path)
21
21
  super().__init__(
22
- image_paths,
23
22
  points,
24
23
  use_gpx_start_time=use_gpx_start_time,
25
24
  offset_time=offset_time,
@@ -27,7 +26,7 @@ class GeotagImagesFromNMEAFile(GeotagImagesFromGPX):
27
26
  )
28
27
 
29
28
 
30
- def get_lat_lon_time_from_nmea(nmea_file: Path) -> T.List[geo.Point]:
29
+ def get_lat_lon_time_from_nmea(nmea_file: Path) -> list[geo.Point]:
31
30
  with nmea_file.open("r") as f:
32
31
  lines = f.readlines()
33
32
  lines = [line.rstrip("\n\r") for line in lines]