mapillary-tools 0.13.3a1__py3-none-any.whl → 0.14.0a2__py3-none-any.whl

mapillary_tools/utils.py

@@ -1,14 +1,15 @@
+from __future__ import annotations
+
 import hashlib
 import os
 import typing as T
+from multiprocessing import Pool
 from pathlib import Path
 
 
 # Use "hashlib._Hash" instead of hashlib._Hash because:
 # AttributeError: module 'hashlib' has no attribute '_Hash'
-def md5sum_fp(
-    fp: T.IO[bytes], md5: T.Optional["hashlib._Hash"] = None
-) -> "hashlib._Hash":
+def md5sum_fp(fp: T.IO[bytes], md5: "hashlib._Hash | None" = None) -> "hashlib._Hash":
     if md5 is None:
         md5 = hashlib.md5()
     while True:
@@ -90,12 +91,12 @@ def filter_video_samples(
 
 
 def find_all_image_samples(
-    image_paths: T.Sequence[Path], video_paths: T.Sequence[Path]
-) -> T.Dict[Path, T.List[Path]]:
+    image_paths: T.Iterable[Path], video_paths: T.Iterable[Path]
+) -> dict[Path, list[Path]]:
     # TODO: not work with the same filenames, e.g. foo/hello.mp4 and bar/hello.mp4
     video_basenames = {path.name: path for path in video_paths}
 
-    image_samples_by_video_path: T.Dict[Path, T.List[Path]] = {}
+    image_samples_by_video_path: dict[Path, list[Path]] = {}
     for image_path in image_paths:
         # If you want to walk an arbitrary filesystem path upwards,
         # it is recommended to first call Path.resolve() so as to resolve symlinks and eliminate “..” components.
@@ -112,7 +113,7 @@ def find_all_image_samples(
 
 
 def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]:
-    resolved_paths: T.Set[Path] = set()
+    resolved_paths: set[Path] = set()
     for p in paths:
         resolved = p.resolve()
         if resolved not in resolved_paths:
@@ -121,10 +122,10 @@ def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]:
 
 
 def find_images(
-    import_paths: T.Sequence[Path],
+    import_paths: T.Iterable[Path],
     skip_subfolders: bool = False,
-) -> T.List[Path]:
-    image_paths: T.List[Path] = []
+) -> list[Path]:
+    image_paths: list[Path] = []
     for path in import_paths:
         if path.is_dir():
             image_paths.extend(
@@ -139,10 +140,10 @@ def find_images(
 
 
 def find_videos(
-    import_paths: T.Sequence[Path],
+    import_paths: T.Iterable[Path],
     skip_subfolders: bool = False,
-) -> T.List[Path]:
-    video_paths: T.List[Path] = []
+) -> list[Path]:
+    video_paths: list[Path] = []
     for path in import_paths:
         if path.is_dir():
             video_paths.extend(
@@ -157,10 +158,10 @@ def find_videos(
 
 
 def find_zipfiles(
-    import_paths: T.Sequence[Path],
+    import_paths: T.Iterable[Path],
     skip_subfolders: bool = False,
-) -> T.List[Path]:
-    zip_paths: T.List[Path] = []
+) -> list[Path]:
+    zip_paths: list[Path] = []
     for path in import_paths:
         if path.is_dir():
             zip_paths.extend(
@@ -174,8 +175,8 @@ def find_zipfiles(
     return list(deduplicate_paths(zip_paths))
 
 
-def find_xml_files(import_paths: T.Sequence[Path]) -> T.List[Path]:
-    xml_paths: T.List[Path] = []
+def find_xml_files(import_paths: T.Iterable[Path]) -> list[Path]:
+    xml_paths: list[Path] = []
     for path in import_paths:
         if path.is_dir():
             # XML could be hidden in hidden folders
@@ -194,3 +195,26 @@ def find_xml_files(import_paths: T.Sequence[Path]) -> T.List[Path]:
 
 def get_file_size(path: Path) -> int:
     return os.path.getsize(path)
+
+
+TMapIn = T.TypeVar("TMapIn")
+TMapOut = T.TypeVar("TMapOut")
+
+
+def mp_map_maybe(
+    func: T.Callable[[TMapIn], TMapOut],
+    iterable: T.Iterable[TMapIn],
+    num_processes: int | None = None,
+) -> T.Generator[TMapOut, None, None]:
+    if num_processes is None:
+        pool_num_processes = None
+        disable_multiprocessing = False
+    else:
+        pool_num_processes = max(num_processes, 1)
+        disable_multiprocessing = num_processes <= 0
+
+    if disable_multiprocessing:
+        yield from map(func, iterable)
+    else:
+        with Pool(processes=pool_num_processes) as pool:
+            yield from pool.imap(func, iterable)

{mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: mapillary_tools
-Version: 0.13.3a1
+Version: 0.14.0a2
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -23,6 +23,7 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

mapillary_tools-0.14.0a2.dist-info/RECORD

@@ -0,0 +1,72 @@
+mapillary_tools/__init__.py,sha256=AdfWnUWJogoEJm_EqofQxlG8NYL78h0B0M3Igr2xQpE,21
+mapillary_tools/api_v4.py,sha256=uvz48tb2HaM1wfomaUR0V2HdId39HoxnQAc1WWgptvo,10091
+mapillary_tools/authenticate.py,sha256=KG3p_7D4D7nUvrGRLGp56_ogLy7JUHzkGSpBw8V70LY,11091
+mapillary_tools/blackvue_parser.py,sha256=Gmz3M1cvBmB2W3Pme6x3w2yuixDpKiTOkQe82iPXr2s,3613
+mapillary_tools/config.py,sha256=uoNW5PnB1a2TW959KmNSYgJ1Y4_trn0xxPHJQ6T-cbI,2633
+mapillary_tools/constants.py,sha256=BVnAlqz2XJfodNXsJfsGIV0L4P_3--oOgw1pR38D-Ls,3770
+mapillary_tools/exceptions.py,sha256=P8vEh3pfOqyzJgOiQiRAcWsZTrcawjic7i4v3k335tM,2411
+mapillary_tools/exif_read.py,sha256=gagyRv06RkZPRHQhyV-5-uQzxHgxOsVqUX2aC8QmCyM,32145
+mapillary_tools/exif_write.py,sha256=Ue0ngLKCfuLQM7X4h5ETjSInUDcyEBWOR6v8ko7DJ1o,8769
+mapillary_tools/exiftool_read.py,sha256=5uatYE9mgbg2d9NAnPfX22nSRBjXhJ9ayMqNMd4QwGM,15779
+mapillary_tools/exiftool_read_video.py,sha256=WXpw7CI7B4daJHRWzzVXAyVgDrFN8nPxQ3UPcVQqRJM,14760
+mapillary_tools/exiftool_runner.py,sha256=J04V0Y70kTiGE2q2m5TKWfORd0hmYNmAsmrZxb34S3s,2316
+mapillary_tools/ffmpeg.py,sha256=tRgdChGHAhHXjZPivDuVImubrJWfSDZDBZ7mSzkeABc,15698
+mapillary_tools/geo.py,sha256=MSfkjSAgTAeuBqncQ8-QXvDuJ7ar7bTt5P_e9HDIgVs,10604
+mapillary_tools/history.py,sha256=8ng2WHXadzbE5gnsoik6gbmRoEXTuGPp997RBiiWy08,1649
+mapillary_tools/ipc.py,sha256=DwWQb9hNshx0bg0Fo5NjY0mXjs-FkbR6tIQmjMgMtmg,1089
+mapillary_tools/process_geotag_properties.py,sha256=f0Up7yNzgLn32VBFMzoMT6SCsdzFUFYmloOsVHmK5k0,13920
+mapillary_tools/process_sequence_properties.py,sha256=LSYyCH1nmEcYWWZKF1bOYh4hqjwXdz4FY9TqQQfW7AU,23418
+mapillary_tools/sample_video.py,sha256=MwTr2bY5RFnOFThNPg3YiyXCXzWOlTXKq7EOri_W3Vw,13880
+mapillary_tools/telemetry.py,sha256=lL6qQbtOZft4DZZrCNK3njlwHT_30zLyYS_YRN5pgHY,1568
+mapillary_tools/types.py,sha256=0O4fzbwwoT6Znp2Ag1ptjzBKGb3E_RhGyqUUZu6gkPY,23078
+mapillary_tools/upload.py,sha256=exQeEh3jpys94k-3QDT-4xoeJ_lAHbQBoy749hbPfnA,24878
+mapillary_tools/upload_api_v4.py,sha256=dTEpQDnV11TyPf5hefd-FVeEfw7WEVYyZcIUJttUEp0,6417
+mapillary_tools/uploader.py,sha256=xPEszuXlgcgCklGfsgL8qN1ozsFuU4rLT4jocMD2dVI,18694
+mapillary_tools/utils.py,sha256=ft-TrqgwWq1a-HpQqnCWIortZbjjTOywllPi8HBA3FU,6642
+mapillary_tools/camm/camm_builder.py,sha256=ub6Z9ijep8zAo1NOlU51Gxk95kQ2vfN58YgVCLmNMRk,9211
+mapillary_tools/camm/camm_parser.py,sha256=aNHP65hNXYQBWBTfhaj_S5XYzmAHhjwcAfGhbm83__o,18043
+mapillary_tools/commands/__init__.py,sha256=41CFrPLGlG3566uhxssEF3TGAtSpADFPPcDMHbViU0E,171
+mapillary_tools/commands/__main__.py,sha256=5gwngScbvrSNF2S1CA5w_qZgxW_kFevu5NQ89U9u1ZE,5046
+mapillary_tools/commands/authenticate.py,sha256=yqtpHMYzkyBrrchj6MARxB0ywUTfqCEOPkMbkyaO9Ks,1344
+mapillary_tools/commands/process.py,sha256=Japc6_P0B_8HzoM8_82P3_YAiyBBaQZXS9TZF46pbMM,9771
+mapillary_tools/commands/process_and_upload.py,sha256=-RB_86a5xKfQ7Ye79dh6yyJQpZg2xnJZAWOJsUNbUtQ,1041
+mapillary_tools/commands/sample_video.py,sha256=jtnrZrsqqv5eYV1chNTas7RhfbeKBqbAUDUNRFjF01w,3253
+mapillary_tools/commands/upload.py,sha256=YnA1iSvNcFeV7kMs1p4TdaPBUWUFYb7X-vFhQeSz0eI,2005
+mapillary_tools/commands/video_process.py,sha256=-wQeeIwWXPmy81HQHam5A0huMLRHknkEFa_V1OwElU4,890
+mapillary_tools/commands/video_process_and_upload.py,sha256=hOyq9L9TuD0JcqFSOOxdCdgsBA1iJ6fu1TtDbsUr8sI,1088
+mapillary_tools/commands/zip.py,sha256=DVQuMLpbstwiy5o4pU_fBvM6eORuFjLeySd80AhHKU0,991
+mapillary_tools/geotag/base.py,sha256=JdLBXHlC22gqmZkyfAykv1osrzitWoCK8RFHdCd5qlc,4864
+mapillary_tools/geotag/factory.py,sha256=2HIOnUtfxsgWn2GF0PbbgHq3Tzyssrl4T39YDMtN42U,9564
+mapillary_tools/geotag/geotag_images_from_exif.py,sha256=a5fDOp4tlEoVNwrOCJBrvSPzu1LbyxL4RNrNDD0tPP0,608
+mapillary_tools/geotag/geotag_images_from_exiftool.py,sha256=YxMFwv3A35Hh9tcGX1B1Si_m6TNep54SL5XFAQadQh0,5302
+mapillary_tools/geotag/geotag_images_from_gpx.py,sha256=b_4xKVUt0HLIjNN-v7ZVddXDZ9cdxAfecB5Dj2X6XQE,6353
+mapillary_tools/geotag/geotag_images_from_gpx_file.py,sha256=HYQkwak32YBDuRrTNiIZOmE3iImCRc22HWWb485WRS4,1121
+mapillary_tools/geotag/geotag_images_from_nmea_file.py,sha256=J8xj6ch9bMPRubJtsRGeb3sb9LyB0ZYy65NPEOVkUe8,1597
+mapillary_tools/geotag/geotag_images_from_video.py,sha256=5_qVgLt_DKl7oQrr9kWxi4iM143_HPAx5Lau-3fB7t4,3136
+mapillary_tools/geotag/geotag_videos_from_exiftool.py,sha256=7chO_tb7ECsBlOkFhiXQ4fhomVPkfBOWtdfzXGpsJes,3168
+mapillary_tools/geotag/geotag_videos_from_gpx.py,sha256=k4a1imUVLg2yDqggCwTSqjvY7jXQd3E-vniBuJj2d4M,1017
+mapillary_tools/geotag/geotag_videos_from_video.py,sha256=T8XS4lVF2Wz4eDXNi5Vt076M5dxjxJXibVrWhqVvErs,863
+mapillary_tools/geotag/options.py,sha256=7Fz0QoCMyZZbhkKvY4HPNWYfnKdQSWE2oRXb7jYZhmA,4336
+mapillary_tools/geotag/utils.py,sha256=0DenLVCLP-GZdk0rCFP0V1tjMGK4XAvYro3I_1RKgCA,1733
+mapillary_tools/geotag/image_extractors/base.py,sha256=XoNrLCbJtd-MN-snbhv6zyr6zBfJRoJkntV0ptrh6qg,358
+mapillary_tools/geotag/image_extractors/exif.py,sha256=cCUegbFqWxjAP4oOP1nZmwoJISWeKgjGO8h_t7nucHs,2079
+mapillary_tools/geotag/image_extractors/exiftool.py,sha256=zokJmf-D2rPvASRJs3dZzEu7j82igpMOr4SE6Z1nsVg,497
+mapillary_tools/geotag/video_extractors/base.py,sha256=_3B37T-m2HGy77avAF8i_kEatIyQqFfBu49qVnx_0PU,357
+mapillary_tools/geotag/video_extractors/exiftool.py,sha256=CB49qZqgmxECma-N5757aEn37xXRbCNRcT4o-Ip7dCw,2312
+mapillary_tools/geotag/video_extractors/gpx.py,sha256=Oycdn74CNiq1j8dAq7O9YiVNTun2mFWRH9VTUn_viRk,4114
+mapillary_tools/geotag/video_extractors/native.py,sha256=XChOyDvrf88Vu8ED-FTT_hzax2FllokanszSW4Q2MEA,5702
+mapillary_tools/gpmf/gpmf_gps_filter.py,sha256=7cg8wEjC1DrujKY76FZguXsaPqTRkG9-t32OeuOJQIc,2755
+mapillary_tools/gpmf/gpmf_parser.py,sha256=uOVXkwJxC3Y2YfTdzUDpt7Bh0pdVqa5u0WUuv7pEJEs,24670
+mapillary_tools/gpmf/gps_filter.py,sha256=zhXkuvr0Dd1bxGTYBwvk6P7xasY_RLuWjHaX7CdBayc,3796
+mapillary_tools/mp4/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mapillary_tools/mp4/construct_mp4_parser.py,sha256=ArJ5C8iWJhHNPFkN048DXh6EwQ8LMfnTMwoNlKV9NoI,17191
+mapillary_tools/mp4/io_utils.py,sha256=KZaJTlgFS27Oh3pcA5MKXYFoCifqgFaEZJyU6lb1jc4,5416
+mapillary_tools/mp4/mp4_sample_parser.py,sha256=0ILTq8M6mXFTI3agKgljpvO9uYa7HXGUGZpdHT8a6ac,11547
+mapillary_tools/mp4/simple_mp4_builder.py,sha256=9TUGk1hzI6mQFN1P30jwHL3dCYz3Zz7rsm8UBvMAqMc,12734
+mapillary_tools/mp4/simple_mp4_parser.py,sha256=g3vvPhBoNu7anhVzC5_XQCV7IwfRWro1vJ6d6GyDkHE,6315
+mapillary_tools-0.14.0a2.dist-info/licenses/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
+mapillary_tools-0.14.0a2.dist-info/METADATA,sha256=UDVbKn5kqizwZmE70erGYDOnSz8sg9CuTd81GHSWtDI,19782
+mapillary_tools-0.14.0a2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+mapillary_tools-0.14.0a2.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
+mapillary_tools-0.14.0a2.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
+mapillary_tools-0.14.0a2.dist-info/RECORD,,

{mapillary_tools-0.13.3a1.dist-info → mapillary_tools-0.14.0a2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

mapillary_tools/geotag/__init__.py

@@ -1 +0,0 @@
-from .. import geo  # noqa: F401

mapillary_tools/geotag/geotag_from_generic.py

@@ -1,22 +0,0 @@
-import abc
-import typing as T
-
-from .. import types
-
-
-class GeotagImagesFromGeneric(abc.ABC):
-    def __init__(self) -> None:
-        pass
-
-    @abc.abstractmethod
-    def to_description(self) -> T.List[types.ImageMetadataOrError]:
-        raise NotImplementedError
-
-
-class GeotagVideosFromGeneric(abc.ABC):
-    def __init__(self) -> None:
-        pass
-
-    @abc.abstractmethod
-    def to_description(self) -> T.List[types.VideoMetadataOrError]:
-        raise NotImplementedError

mapillary_tools/geotag/geotag_images_from_exiftool_both_image_and_video.py

@@ -1,93 +0,0 @@
-import logging
-import typing as T
-from pathlib import Path
-
-from .. import exiftool_read, types, utils
-from . import (
-    geotag_images_from_exiftool,
-    geotag_images_from_video,
-    geotag_videos_from_exiftool_video,
-)
-from .geotag_from_generic import GeotagImagesFromGeneric
-
-
-LOG = logging.getLogger(__name__)
-
-
-class GeotagImagesFromExifToolBothImageAndVideo(GeotagImagesFromGeneric):
-    def __init__(
-        self,
-        image_paths: T.Sequence[Path],
-        xml_path: Path,
-        offset_time: float = 0.0,
-        num_processes: T.Optional[int] = None,
-    ):
-        self.image_paths = image_paths
-        self.xml_path = xml_path
-        self.offset_time = offset_time
-        self.num_processes = num_processes
-        super().__init__()
-
-    def to_description(self) -> T.List[types.ImageMetadataOrError]:
-        # will return this list
-        final_image_metadatas: T.List[types.ImageMetadataOrError] = []
-
-        # find the images that can be geotagged from EXIF
-        image_metadatas_from_exiftool = (
-            geotag_images_from_exiftool.GeotagImagesFromExifTool(
-                self.image_paths,
-                self.xml_path,
-                num_processes=self.num_processes,
-            ).to_description()
-        )
-
-        # find all video paths in self.xml_path
-        rdf_description_by_path = exiftool_read.index_rdf_description_by_path(
-            [self.xml_path]
-        )
-        video_paths = utils.find_videos(
-            [Path(pathstr) for pathstr in rdf_description_by_path.keys()],
-            skip_subfolders=True,
-        )
-
-        # will try to geotag these error metadatas from video later
-        error_metadata_by_image_path = {}
-        for image_metadata in image_metadatas_from_exiftool:
-            if isinstance(image_metadata, types.ErrorMetadata):
-                error_metadata_by_image_path[image_metadata.filename] = image_metadata
-            else:
-                final_image_metadatas.append(image_metadata)
-
-        maybe_image_samples = list(error_metadata_by_image_path.keys())
-
-        # find all video paths that have sample images
-        video_paths_with_image_samples = list(
-            utils.find_all_image_samples(maybe_image_samples, video_paths).keys()
-        )
-
-        video_metadatas = (
-            geotag_videos_from_exiftool_video.GeotagVideosFromExifToolVideo(
-                video_paths_with_image_samples,
-                self.xml_path,
-                num_processes=self.num_processes,
-            ).to_description()
-        )
-
-        image_metadatas_from_video = geotag_images_from_video.GeotagImagesFromVideo(
-            maybe_image_samples,
-            video_metadatas,
-            offset_time=self.offset_time,
-            num_processes=self.num_processes,
-        ).to_description()
-        final_image_metadatas.extend(image_metadatas_from_video)
-
-        # add back error metadatas that can not be geotagged at all
-        actual_image_sample_paths = set(
-            image_metadata.filename for image_metadata in image_metadatas_from_video
-        )
-        for path, error_metadata in error_metadata_by_image_path.items():
-            if path not in actual_image_sample_paths:
-                final_image_metadatas.append(error_metadata)
-
-        assert len(final_image_metadatas) <= len(self.image_paths)
-        return final_image_metadatas

mapillary_tools/geotag/geotag_videos_from_exiftool_video.py

@@ -1,145 +0,0 @@
-import logging
-import typing as T
-import xml.etree.ElementTree as ET
-from multiprocessing import Pool
-from pathlib import Path
-
-from tqdm import tqdm
-
-from .. import exceptions, exiftool_read, geo, types, utils
-from ..exiftool_read_video import ExifToolReadVideo
-from ..telemetry import GPSPoint
-from . import gpmf_gps_filter, utils as video_utils
-from .geotag_from_generic import GeotagVideosFromGeneric
-
-LOG = logging.getLogger(__name__)
-_DESCRIPTION_TAG = "rdf:Description"
-
-
-class GeotagVideosFromExifToolVideo(GeotagVideosFromGeneric):
-    def __init__(
-        self,
-        video_paths: T.Sequence[Path],
-        xml_path: Path,
-        num_processes: T.Optional[int] = None,
-    ):
-        self.video_paths = video_paths
-        self.xml_path = xml_path
-        self.num_processes = num_processes
-        super().__init__()
-
-    @staticmethod
-    def geotag_video(element: ET.Element) -> types.VideoMetadataOrError:
-        video_path = exiftool_read.find_rdf_description_path(element)
-        assert video_path is not None, "must find the path from the element"
-
-        try:
-            exif = ExifToolReadVideo(ET.ElementTree(element))
-
-            points = exif.extract_gps_track()
-
-            if not points:
-                raise exceptions.MapillaryVideoGPSNotFoundError(
-                    "No GPS data found from the video"
-                )
-
-            points = geo.extend_deduplicate_points(points)
-            assert points, "must have at least one point"
-
-            if all(isinstance(p, GPSPoint) for p in points):
-                points = T.cast(
-                    T.List[geo.Point],
-                    gpmf_gps_filter.remove_noisy_points(
-                        T.cast(T.List[GPSPoint], points)
-                    ),
-                )
-                if not points:
-                    raise exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-
-            stationary = video_utils.is_video_stationary(
-                geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
-            )
-
-            if stationary:
-                raise exceptions.MapillaryStationaryVideoError("Stationary video")
-
-            video_metadata = types.VideoMetadata(
-                video_path,
-                md5sum=None,
-                filesize=utils.get_file_size(video_path),
-                filetype=types.FileType.VIDEO,
-                points=points,
-                make=exif.extract_make(),
-                model=exif.extract_model(),
-            )
-
-            LOG.debug("Calculating MD5 checksum for %s", str(video_metadata.filename))
-
-            video_metadata.update_md5sum()
-
-        except Exception as ex:
-            if not isinstance(ex, exceptions.MapillaryDescriptionError):
-                LOG.warning(
-                    "Failed to geotag video %s: %s",
-                    video_path,
-                    str(ex),
-                    exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
-                )
-            return types.describe_error_metadata(
-                ex, video_path, filetype=types.FileType.VIDEO
-            )
-
-        return video_metadata
-
-    def to_description(self) -> T.List[types.VideoMetadataOrError]:
-        rdf_description_by_path = exiftool_read.index_rdf_description_by_path(
-            [self.xml_path]
-        )
-
-        error_metadatas: T.List[types.ErrorMetadata] = []
-        rdf_descriptions: T.List[ET.Element] = []
-        for path in self.video_paths:
-            rdf_description = rdf_description_by_path.get(
-                exiftool_read.canonical_path(path)
-            )
-            if rdf_description is None:
-                exc = exceptions.MapillaryEXIFNotFoundError(
-                    f"The {_DESCRIPTION_TAG} XML element for the video not found"
-                )
-                error_metadatas.append(
-                    types.describe_error_metadata(
-                        exc, path, filetype=types.FileType.VIDEO
-                    )
-                )
-            else:
-                rdf_descriptions.append(rdf_description)
-
-        if self.num_processes is None:
-            num_processes = self.num_processes
-            disable_multiprocessing = False
-        else:
-            num_processes = max(self.num_processes, 1)
-            disable_multiprocessing = self.num_processes <= 0
-
-        with Pool(processes=num_processes) as pool:
-            video_metadatas_iter: T.Iterator[types.VideoMetadataOrError]
-            if disable_multiprocessing:
-                video_metadatas_iter = map(
-                    GeotagVideosFromExifToolVideo.geotag_video, rdf_descriptions
-                )
-            else:
-                video_metadatas_iter = pool.imap(
-                    GeotagVideosFromExifToolVideo.geotag_video,
-                    rdf_descriptions,
-                )
-            video_metadata_or_errors = list(
-                tqdm(
-                    video_metadatas_iter,
-                    desc="Extracting GPS tracks from ExifTool XML",
-                    unit="videos",
-                    disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                    total=len(self.video_paths),
-                )
-            )
-
-        return error_metadatas + video_metadata_or_errors

mapillary_tools/video_data_extraction/cli_options.py

@@ -1,22 +0,0 @@
-import typing as T
-from pathlib import Path
-
-
-known_parser_options = ["source", "pattern", "exiftool_path"]
-
-
-class CliParserOptions(T.TypedDict, total=False):
-    source: str
-    pattern: T.Optional[str]
-    exiftool_path: T.Optional[Path]
-
-
-class CliOptions(T.TypedDict, total=False):
-    paths: T.Sequence[Path]
-    recursive: bool
-    geotag_sources_options: T.Sequence[CliParserOptions]
-    geotag_source_path: Path
-    exiftool_path: Path
-    num_processes: int
-    device_make: T.Optional[str]
-    device_model: T.Optional[str]

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -1,176 +0,0 @@
-import logging
-import typing as T
-from multiprocessing import Pool
-from pathlib import Path
-
-import tqdm
-
-from .. import exceptions, geo, utils
-from ..geotag import gpmf_gps_filter, utils as video_utils
-from ..telemetry import GPSPoint
-from ..types import (
-    ErrorMetadata,
-    FileType,
-    MetadataOrError,
-    VideoMetadata,
-    VideoMetadataOrError,
-)
-from . import video_data_parser_factory
-from .cli_options import CliOptions
-from .extractors.base_parser import BaseParser
-
-
-LOG = logging.getLogger(__name__)
-
-
-class VideoDataExtractor:
-    options: CliOptions
-
-    def __init__(self, options: CliOptions) -> None:
-        self.options = options
-
-    def process(self) -> T.List[MetadataOrError]:
-        paths = self.options["paths"]
-        self._check_paths(paths)
-        video_files = utils.find_videos(paths)
-        self._check_sources_cardinality(video_files)
-
-        num_processes = self.options["num_processes"] or None
-        with Pool(processes=num_processes) as pool:
-            if num_processes == 1:
-                iter: T.Iterator[VideoMetadataOrError] = map(
-                    self.process_file, video_files
-                )
-            else:
-                iter = pool.imap(self.process_file, video_files)
-
-            video_metadata_or_errors = list(
-                tqdm.tqdm(
-                    iter,
-                    desc="Extracting GPS tracks",
-                    unit="videos",
-                    disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                    total=len(video_files),
-                )
-            )
-
-        return video_metadata_or_errors
-
-    def process_file(self, file: Path) -> VideoMetadataOrError:
-        parsers = video_data_parser_factory.make_parsers(file, self.options)
-        points: T.Sequence[geo.Point] = []
-        make = self.options["device_make"]
-        model = self.options["device_model"]
-
-        ex: T.Optional[Exception]
-        for parser in parsers:
-            log_vars = {
-                "filename": file,
-                "parser": parser.parser_label,
-                "source": parser.geotag_source_path,
-            }
-            try:
-                if not points:
-                    points = self._extract_points(parser, log_vars)
-                if not model:
-                    model = parser.extract_model()
-                if not make:
-                    make = parser.extract_make()
-            except Exception as e:
-                ex = e
-                LOG.warning(
-                    '%(filename)s: Exception for parser %(parser)s while processing source %(source)s: "%(e)s"',
-                    {**log_vars, "e": e},
-                )
-
-        # After trying all parsers, return the points if we found any, otherwise
-        # the last exception thrown or a default one.
-        # Note that if we have points, we return them, regardless of exceptions
-        # with make or model.
-        if points:
-            video_metadata = VideoMetadata(
-                filename=file,
-                filetype=FileType.VIDEO,
-                md5sum=None,
-                filesize=utils.get_file_size(file),
-                points=points,
-                make=make,
-                model=model,
-            )
-            video_metadata.update_md5sum()
-            return video_metadata
-        else:
-            return ErrorMetadata(
-                filename=file,
-                error=(
-                    ex
-                    if ex
-                    else exceptions.MapillaryVideoGPSNotFoundError(
-                        "No GPS data found from the video"
-                    )
-                ),
-                filetype=FileType.VIDEO,
-            )
-
-    def _extract_points(
-        self, parser: BaseParser, log_vars: T.Dict
-    ) -> T.Sequence[geo.Point]:
-        points = parser.extract_points()
-        if points:
-            LOG.debug(
-                "%(filename)s: %(points)d points extracted by parser %(parser)s from file %(source)s}",
-                {**log_vars, "points": len(points)},
-            )
-
-        return self._sanitize_points(points)
-
-    @staticmethod
-    def _check_paths(import_paths: T.Sequence[Path]):
-        for path in import_paths:
-            if not path.is_file() and not path.is_dir():
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Import file or directory not found: {path}"
-                )
-
-    def _check_sources_cardinality(self, files: T.Sequence[Path]):
-        if len(files) > 1:
-            for parser_opts in self.options["geotag_sources_options"]:
-                pattern = parser_opts.get("pattern")
-                if pattern and "%" not in pattern:
-                    raise exceptions.MapillaryUserError(
-                        "Multiple video files found: Geotag source pattern for source %s must include filename placeholders",
-                        parser_opts["source"],
-                    )
-
-    @staticmethod
-    def _sanitize_points(points: T.Sequence[geo.Point]) -> T.Sequence[geo.Point]:
-        """
-        Deduplicates points, when possible removes noisy ones, and checks
-        against stationary videos
-        """
-
-        if not points:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found in the given sources"
-            )
-
-        points = geo.extend_deduplicate_points(points)
-
-        if all(isinstance(p, GPSPoint) for p in points):
-            points = T.cast(
-                T.Sequence[geo.Point],
-                gpmf_gps_filter.remove_noisy_points(
-                    T.cast(T.Sequence[GPSPoint], points)
-                ),
-            )
-            if not points:
-                raise exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-
-        stationary = video_utils.is_video_stationary(
-            geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
-        )
-
-        if stationary:
-            raise exceptions.MapillaryStationaryVideoError("Stationary video")
-
-        return points