mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

mapillary_tools/exiftool_read_video.py

@@ -5,6 +5,7 @@ import typing as T
 import xml.etree.ElementTree as ET
 
 from . import exif_read, exiftool_read, geo
+from .telemetry import GPSFix, GPSPoint
 
 
 MAX_TRACK_ID = 10
@@ -87,7 +88,7 @@ def _aggregate_gps_track(
     alt_tag: T.Optional[str] = None,
     direction_tag: T.Optional[str] = None,
     ground_speed_tag: T.Optional[str] = None,
-) -> T.List[geo.PointWithFix]:
+) -> T.List[GPSPoint]:
     """
     Aggregate all GPS data by the tags.
     It requires lat, lon to be present, and their lengths must match.
@@ -173,15 +174,16 @@ def _aggregate_gps_track(
         if timestamp is None or lon is None or lat is None:
             continue
         track.append(
-            geo.PointWithFix(
+            GPSPoint(
                 time=timestamp,
                 lon=lon,
                 lat=lat,
                 alt=alt,
                 angle=direction,
-                gps_fix=None,
-                gps_precision=None,
-                gps_ground_speed=ground_speed,
+                epoch_time=None,
+                fix=None,
+                precision=None,
+                ground_speed=ground_speed,
             )
         )
 
@@ -230,8 +232,8 @@ def _aggregate_gps_track_by_sample_time(
     ground_speed_tag: T.Optional[str] = None,
     gps_fix_tag: T.Optional[str] = None,
     gps_precision_tag: T.Optional[str] = None,
-) -> T.List[geo.PointWithFix]:
-    track: T.List[geo.PointWithFix] = []
+) -> T.List[GPSPoint]:
+    track: T.List[GPSPoint] = []
 
     expanded_gps_fix_tag = None
     if gps_fix_tag is not None:
@@ -249,7 +251,7 @@ def _aggregate_gps_track_by_sample_time(
             gps_fix_texts = texts_by_tag.get(expanded_gps_fix_tag)
             if gps_fix_texts:
                 try:
-                    gps_fix = geo.GPSFix(int(gps_fix_texts[0]))
+                    gps_fix = GPSFix(int(gps_fix_texts[0]))
                 except ValueError:
                     gps_fix = None
 
@@ -280,7 +282,7 @@ def _aggregate_gps_track_by_sample_time(
             for idx, point in enumerate(points):
                 point.time = sample_time + idx * avg_timedelta
             track.extend(
-                dataclasses.replace(point, gps_fix=gps_fix, gps_precision=gps_precision)
+                dataclasses.replace(point, fix=gps_fix, precision=gps_precision)
                 for point in points
             )
 
@@ -355,7 +357,7 @@ class ExifToolReadVideo:
         _, model = self._extract_make_and_model()
         return model
 
-    def _extract_gps_track_from_track(self) -> T.List[geo.PointWithFix]:
+    def _extract_gps_track_from_track(self) -> T.List[GPSPoint]:
         for track_id in range(1, MAX_TRACK_ID + 1):
             track_ns = f"Track{track_id}"
             if self._all_tags_exists(
@@ -397,7 +399,7 @@ class ExifToolReadVideo:
 
     def _extract_gps_track_from_quicktime(
         self, namespace: str = "QuickTime"
-    ) -> T.List[geo.PointWithFix]:
+    ) -> T.List[GPSPoint]:
         if not self._all_tags_exists(
             {
                 expand_tag(f"{namespace}:GPSDateTime"),

mapillary_tools/ffmpeg.py

@@ -202,10 +202,10 @@ class FFMPEG:
             return "0"
 
         if length == 1:
-            return f"eq(n\\,{ sorted_frame_indices[0] })"
+            return f"eq(n\\,{sorted_frame_indices[0]})"
 
         middle = length // 2
-        return f"if(lt(n\\,{ sorted_frame_indices[middle] })\\,{ self.generate_binary_search(sorted_frame_indices[:middle]) }\\,{ self.generate_binary_search(sorted_frame_indices[middle:]) })"
+        return f"if(lt(n\\,{sorted_frame_indices[middle]})\\,{self.generate_binary_search(sorted_frame_indices[:middle])}\\,{self.generate_binary_search(sorted_frame_indices[middle:])})"
 
     def extract_specified_frames(
         self,

mapillary_tools/geo.py

@@ -6,7 +6,6 @@ import datetime
 import itertools
 import math
 import typing as T
-from enum import Enum, unique
 
 WGS84_a = 6378137.0
 WGS84_a_SQ = WGS84_a**2
@@ -32,45 +31,6 @@ class Point:
     angle: T.Optional[float]
 
 
-@unique
-class GPSFix(Enum):
-    NO_FIX = 0
-    FIX_2D = 2
-    FIX_3D = 3
-
-
-@dataclasses.dataclass
-class PointWithFix(Point):
-    gps_fix: T.Optional[GPSFix]
-    gps_precision: T.Optional[float]
-    gps_ground_speed: T.Optional[float]
-
-
-def _ecef_from_lla_DEPRECATED(
-    lat: float, lon: float, alt: float
-) -> T.Tuple[float, float, float]:
-    """
-    Deprecated because it is slow. Keep here for reference and comparison.
-    Use _ecef_from_lla2 instead.
-
-    Compute ECEF XYZ from latitude, longitude and altitude.
-
-    All using the WGS94 model.
-    Altitude is the distance to the WGS94 ellipsoid.
-    Check results here http://www.oc.nps.edu/oc2902w/coord/llhxyz.htm
-
-    """
-    a2 = WGS84_a**2
-    b2 = WGS84_b**2
-    lat = math.radians(lat)
-    lon = math.radians(lon)
-    L = 1.0 / math.sqrt(a2 * math.cos(lat) ** 2 + b2 * math.sin(lat) ** 2)
-    x = (a2 * L + alt) * math.cos(lat) * math.cos(lon)
-    y = (a2 * L + alt) * math.cos(lat) * math.sin(lon)
-    z = (b2 * L + alt) * math.sin(lat)
-    return x, y, z
-
-
 def _ecef_from_lla2(lat: float, lon: float) -> T.Tuple[float, float, float]:
     """
     Compute ECEF XYZ from latitude, longitude and altitude.
@@ -172,20 +132,6 @@ def pairwise(iterable: T.Iterable[_IT]) -> T.Iterable[T.Tuple[_IT, _IT]]:
     return zip(a, b)
 
 
-def group_every(
-    iterable: T.Iterable[_IT], n: int
-) -> T.Generator[T.Generator[_IT, None, None], None, None]:
-    """
-    Return a generator that divides the iterable into groups by N.
-    """
-
-    if not (0 < n):
-        raise ValueError("expect 0 < n but got {0}".format(n))
-
-    for _, group in itertools.groupby(enumerate(iterable), key=lambda t: t[0] // n):
-        yield (item for _, item in group)
-
-
 def as_unix_time(dt: T.Union[datetime.datetime, int, float]) -> float:
     if isinstance(dt, (int, float)):
         return dt

mapillary_tools/geotag/blackvue_parser.py

@@ -7,7 +7,7 @@ import typing as T
 import pynmea2
 
 from .. import geo
-from . import simple_mp4_parser
+from ..mp4 import simple_mp4_parser as sparser
 
 
 LOG = logging.getLogger(__name__)
@@ -55,8 +55,8 @@ def _parse_gps_box(gps_data: bytes) -> T.Generator[geo.Point, None, None]:
 
 def extract_camera_model(fp: T.BinaryIO) -> str:
     try:
-        cprt_bytes = simple_mp4_parser.parse_mp4_data_first(fp, [b"free", b"cprt"])
-    except simple_mp4_parser.ParsingError:
+        cprt_bytes = sparser.parse_mp4_data_first(fp, [b"free", b"cprt"])
+    except sparser.ParsingError:
         return ""
 
     if cprt_bytes is None:
@@ -91,7 +91,7 @@ def extract_camera_model(fp: T.BinaryIO) -> str:
 
 
 def extract_points(fp: T.BinaryIO) -> T.Optional[T.List[geo.Point]]:
-    gps_data = simple_mp4_parser.parse_mp4_data_first(fp, [b"free", b"gps "])
+    gps_data = sparser.parse_mp4_data_first(fp, [b"free", b"gps "])
     if gps_data is None:
         return None
 

mapillary_tools/geotag/geotag_images_from_exif.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from tqdm import tqdm
 
-from .. import exceptions, exif_write, geo, types
+from .. import exceptions, exif_write, geo, types, utils
 from ..exif_read import ExifRead, ExifReadABC
 from .geotag_from_generic import GeotagImagesFromGeneric
 
@@ -64,6 +64,7 @@ class GeotagImagesFromEXIF(GeotagImagesFromGeneric):
         image_metadata = types.ImageMetadata(
             filename=image_path,
             md5sum=None,
+            filesize=utils.get_file_size(image_path),
             time=geo.as_unix_time(capture_time),
             lat=lat,
             lon=lon,

mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py

@@ -48,7 +48,6 @@ class GeotagImagesFromExifToolBothImageAndVideo(GeotagImagesFromGeneric):
         video_paths = utils.find_videos(
             [Path(pathstr) for pathstr in rdf_description_by_path.keys()],
             skip_subfolders=True,
-            check_file_suffix=True,
         )
 
         # will try to geotag these error metadatas from video later

mapillary_tools/geotag/geotag_images_from_gpx_file.py

@@ -25,7 +25,13 @@ class GeotagImagesFromGPXFile(GeotagImagesFromGeneric):
         num_processes: T.Optional[int] = None,
     ):
         super().__init__()
-        tracks = parse_gpx(source_path)
+        try:
+            tracks = parse_gpx(source_path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {source_path}: {ex.__class__.__name__}: {ex}"
+            )
+
         if 1 < len(tracks):
             LOG.warning(
                 "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",

mapillary_tools/geotag/geotag_videos_from_exiftool_video.py

@@ -6,8 +6,9 @@ from pathlib import Path
 
 from tqdm import tqdm
 
-from .. import exceptions, exiftool_read, geo, types
+from .. import exceptions, exiftool_read, geo, types, utils
 from ..exiftool_read_video import ExifToolReadVideo
+from ..telemetry import GPSPoint
 from . import gpmf_gps_filter, utils as video_utils
 from .geotag_from_generic import GeotagVideosFromGeneric
 
@@ -45,11 +46,11 @@ class GeotagVideosFromExifToolVideo(GeotagVideosFromGeneric):
             points = geo.extend_deduplicate_points(points)
             assert points, "must have at least one point"
 
-            if all(isinstance(p, geo.PointWithFix) for p in points):
+            if all(isinstance(p, GPSPoint) for p in points):
                 points = T.cast(
                     T.List[geo.Point],
                     gpmf_gps_filter.remove_noisy_points(
-                        T.cast(T.List[geo.PointWithFix], points)
+                        T.cast(T.List[GPSPoint], points)
                     ),
                 )
                 if not points:
@@ -65,6 +66,7 @@ class GeotagVideosFromExifToolVideo(GeotagVideosFromGeneric):
             video_metadata = types.VideoMetadata(
                 video_path,
                 md5sum=None,
+                filesize=utils.get_file_size(video_path),
                 filetype=types.FileType.VIDEO,
                 points=points,
                 make=exif.extract_make(),

mapillary_tools/geotag/geotag_videos_from_video.py

@@ -6,15 +6,11 @@ from pathlib import Path
 
 from tqdm import tqdm
 
-from .. import exceptions, geo, types
-from . import (
-    blackvue_parser,
-    camm_parser,
-    gpmf_gps_filter,
-    gpmf_parser,
-    simple_mp4_parser as parser,
-    utils as video_utils,
-)
+from .. import exceptions, geo, types, utils
+from ..camm import camm_parser
+from ..mp4 import simple_mp4_parser as sparser
+from ..telemetry import GPSPoint
+from . import blackvue_parser, gpmf_gps_filter, gpmf_parser, utils as video_utils
 from .geotag_from_generic import GeotagVideosFromGeneric
 
 LOG = logging.getLogger(__name__)
@@ -77,7 +73,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
             with video_path.open("rb") as fp:
                 try:
                     points = camm_parser.extract_points(fp)
-                except parser.ParsingError:
+                except sparser.ParsingError:
                     points = None
 
                 if points is not None:
@@ -86,6 +82,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        filesize=utils.get_file_size(video_path),
                         filetype=types.FileType.CAMM,
                         points=points,
                         make=make,
@@ -100,7 +97,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
             with video_path.open("rb") as fp:
                 try:
                     points_with_fix = gpmf_parser.extract_points(fp)
-                except parser.ParsingError:
+                except sparser.ParsingError:
                     points_with_fix = None
 
                 if points_with_fix is not None:
@@ -109,6 +106,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        filesize=utils.get_file_size(video_path),
                         filetype=types.FileType.GOPRO,
                         points=T.cast(T.List[geo.Point], points_with_fix),
                         make=make,
@@ -123,7 +121,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
             with video_path.open("rb") as fp:
                 try:
                     points = blackvue_parser.extract_points(fp)
-                except parser.ParsingError:
+                except sparser.ParsingError:
                     points = None
 
                 if points is not None:
@@ -132,6 +130,7 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        filesize=utils.get_file_size(video_path),
                         filetype=types.FileType.BLACKVUE,
                         points=points,
                         make=make,
@@ -160,11 +159,11 @@ class GeotagVideosFromVideo(GeotagVideosFromGeneric):
             video_metadata.points = geo.extend_deduplicate_points(video_metadata.points)
             assert video_metadata.points, "must have at least one point"
 
-            if all(isinstance(p, geo.PointWithFix) for p in video_metadata.points):
+            if all(isinstance(p, GPSPoint) for p in video_metadata.points):
                 video_metadata.points = T.cast(
                     T.List[geo.Point],
                     gpmf_gps_filter.remove_noisy_points(
-                        T.cast(T.List[geo.PointWithFix], video_metadata.points)
+                        T.cast(T.List[GPSPoint], video_metadata.points)
                     ),
                 )
                 if not video_metadata.points:

mapillary_tools/geotag/gpmf_gps_filter.py

@@ -2,6 +2,7 @@ import logging
 import typing as T
 
 from .. import constants, geo
+from ..telemetry import GPSPoint
 from . import gps_filter
 
 """
@@ -13,8 +14,8 @@ LOG = logging.getLogger(__name__)
 
 
 def remove_outliers(
-    sequence: T.Sequence[geo.PointWithFix],
-) -> T.Sequence[geo.PointWithFix]:
+    sequence: T.Sequence[GPSPoint],
+) -> T.Sequence[GPSPoint]:
     distances = [
         geo.gps_distance((left.lat, left.lon), (right.lat, right.lon))
         for left, right in geo.pairwise(sequence)
@@ -37,9 +38,7 @@ def remove_outliers(
         "Split to %d sequences with max distance %f", len(sequences), max_distance
     )
 
-    ground_speeds = [
-        p.gps_ground_speed for p in sequence if p.gps_ground_speed is not None
-    ]
+    ground_speeds = [p.ground_speed for p in sequence if p.ground_speed is not None]
     if len(ground_speeds) < 2:
         return sequence
 
@@ -50,20 +49,20 @@ def remove_outliers(
     )
 
     return T.cast(
-        T.List[geo.PointWithFix],
+        T.List[GPSPoint],
         gps_filter.find_majority(merged.values()),
     )
 
 
 def remove_noisy_points(
-    sequence: T.Sequence[geo.PointWithFix],
-) -> T.Sequence[geo.PointWithFix]:
+    sequence: T.Sequence[GPSPoint],
+) -> T.Sequence[GPSPoint]:
     num_points = len(sequence)
     sequence = [
         p
         for p in sequence
         # include points **without** GPS fix
-        if p.gps_fix is None or p.gps_fix.value in constants.GOPRO_GPS_FIXES
+        if p.fix is None or p.fix.value in constants.GOPRO_GPS_FIXES
     ]
     if len(sequence) < num_points:
         LOG.debug(
@@ -77,7 +76,7 @@ def remove_noisy_points(
         p
         for p in sequence
         # include points **without** precision
-        if p.gps_precision is None or p.gps_precision <= constants.GOPRO_MAX_DOP100
+        if p.precision is None or p.precision <= constants.GOPRO_MAX_DOP100
     ]
     if len(sequence) < num_points:
         LOG.debug(