mapillary-tools 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

mapillary_tools/upload_api_v4.py

@@ -8,13 +8,12 @@ import typing as T
 
 import requests
 
+from .api_v4 import MAPILLARY_GRAPH_API_ENDPOINT, request_get, request_post
+
 LOG = logging.getLogger(__name__)
 MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
     "MAPILLARY_UPLOAD_ENDPOINT", "https://rupload.facebook.com/mapillary_public_uploads"
 )
-MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
-    "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
-)
 DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
 # (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
@@ -93,7 +92,7 @@ class UploadService:
         }
         url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
         LOG.debug("GET %s", url)
-        resp = requests.get(
+        resp = request_get(
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
@@ -134,7 +133,7 @@ class UploadService:
             }
             url = f"{MAPILLARY_UPLOAD_ENDPOINT}/{self.session_key}"
             LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-            resp = requests.post(
+            resp = request_post(
                 url,
                 headers=headers,
                 data=chunk,
@@ -154,9 +153,9 @@ class UploadService:
             if not chunk:
                 break
 
-        assert (
-            offset == self.entity_size
-        ), f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        assert offset == self.entity_size, (
+            f"Offset ends at {offset} but the entity size is {self.entity_size}"
+        )
 
         payload = resp.json()
         try:
@@ -180,7 +179,7 @@ class UploadService:
         url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
 
         LOG.debug("POST %s HEADERS %s", url, json.dumps(_sanitize_headers(headers)))
-        resp = requests.post(
+        resp = request_post(
             url,
             headers=headers,
             json=data,

mapillary_tools/utils.py

@@ -123,7 +123,6 @@ def deduplicate_paths(paths: T.Iterable[Path]) -> T.Generator[Path, None, None]:
 def find_images(
     import_paths: T.Sequence[Path],
     skip_subfolders: bool = False,
-    check_file_suffix: bool = False,
 ) -> T.List[Path]:
     image_paths: T.List[Path] = []
     for path in import_paths:
@@ -134,10 +133,7 @@ def find_images(
                 if is_image_file(file)
             )
         else:
-            if check_file_suffix:
-                if is_image_file(path):
-                    image_paths.append(path)
-            else:
+            if is_image_file(path):
                 image_paths.append(path)
     return list(deduplicate_paths(image_paths))
 
@@ -145,7 +141,6 @@ def find_images(
 def find_videos(
     import_paths: T.Sequence[Path],
     skip_subfolders: bool = False,
-    check_file_suffix: bool = False,
 ) -> T.List[Path]:
     video_paths: T.List[Path] = []
     for path in import_paths:
@@ -156,10 +151,7 @@ def find_videos(
                 if is_video_file(file)
             )
         else:
-            if check_file_suffix:
-                if is_video_file(path):
-                    video_paths.append(path)
-            else:
+            if is_video_file(path):
                 video_paths.append(path)
     return list(deduplicate_paths(video_paths))
 
@@ -167,7 +159,6 @@ def find_videos(
 def find_zipfiles(
     import_paths: T.Sequence[Path],
     skip_subfolders: bool = False,
-    check_file_suffix: bool = False,
 ) -> T.List[Path]:
     zip_paths: T.List[Path] = []
     for path in import_paths:
@@ -178,10 +169,7 @@ def find_zipfiles(
                 if file.suffix.lower() in [".zip"]
             )
         else:
-            if check_file_suffix:
-                if path.suffix.lower() in [".zip"]:
-                    zip_paths.append(path)
-            else:
+            if path.suffix.lower() in [".zip"]:
                 zip_paths.append(path)
     return list(deduplicate_paths(zip_paths))
 
@@ -199,5 +187,10 @@ def find_xml_files(import_paths: T.Sequence[Path]) -> T.List[Path]:
                 if file.suffix.lower() in [".xml"]
             )
         else:
-            xml_paths.append(path)
+            if path.suffix.lower() in [".xml"]:
+                xml_paths.append(path)
     return list(deduplicate_paths(xml_paths))
+
+
+def get_file_size(path: Path) -> int:
+    return os.path.getsize(path)

mapillary_tools/video_data_extraction/cli_options.py

@@ -20,4 +20,3 @@ class CliOptions(T.TypedDict, total=False):
     num_processes: int
     device_make: T.Optional[str]
     device_model: T.Optional[str]
-    check_file_suffix: bool

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -5,20 +5,19 @@ from pathlib import Path
 
 import tqdm
 
-import mapillary_tools.geotag.utils as video_utils
-
-from mapillary_tools import exceptions, geo, utils
-from mapillary_tools.geotag import gpmf_gps_filter
-from mapillary_tools.types import (
+from .. import exceptions, geo, utils
+from ..geotag import gpmf_gps_filter, utils as video_utils
+from ..telemetry import GPSPoint
+from ..types import (
     ErrorMetadata,
     FileType,
     MetadataOrError,
     VideoMetadata,
     VideoMetadataOrError,
 )
-from mapillary_tools.video_data_extraction import video_data_parser_factory
-from mapillary_tools.video_data_extraction.cli_options import CliOptions
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from . import video_data_parser_factory
+from .cli_options import CliOptions
+from .extractors.base_parser import BaseParser
 
 
 LOG = logging.getLogger(__name__)
@@ -33,9 +32,7 @@ class VideoDataExtractor:
     def process(self) -> T.List[MetadataOrError]:
         paths = self.options["paths"]
         self._check_paths(paths)
-        video_files = utils.find_videos(
-            paths, check_file_suffix=self.options["check_file_suffix"]
-        )
+        video_files = utils.find_videos(paths)
         self._check_sources_cardinality(video_files)
 
         num_processes = self.options["num_processes"] or None
@@ -81,7 +78,7 @@ class VideoDataExtractor:
                     make = parser.extract_make()
             except Exception as e:
                 ex = e
-                LOG.warn(
+                LOG.warning(
                     '%(filename)s: Exception for parser %(parser)s while processing source %(source)s: "%(e)s"',
                     {**log_vars, "e": e},
                 )
@@ -95,6 +92,7 @@ class VideoDataExtractor:
                 filename=file,
                 filetype=FileType.VIDEO,
                 md5sum=None,
+                filesize=utils.get_file_size(file),
                 points=points,
                 make=make,
                 model=model,
@@ -124,12 +122,7 @@ class VideoDataExtractor:
                 {**log_vars, "points": len(points)},
             )
 
-        points = self._sanitize_points(points)
-
-        if parser.must_rebase_times_to_zero:
-            points = self._rebase_times(points)
-
-        return points
+        return self._sanitize_points(points)
 
     @staticmethod
     def _check_paths(import_paths: T.Sequence[Path]):
@@ -163,11 +156,11 @@ class VideoDataExtractor:
 
         points = geo.extend_deduplicate_points(points)
 
-        if all(isinstance(p, geo.PointWithFix) for p in points):
+        if all(isinstance(p, GPSPoint) for p in points):
             points = T.cast(
                 T.Sequence[geo.Point],
                 gpmf_gps_filter.remove_noisy_points(
-                    T.cast(T.Sequence[geo.PointWithFix], points)
+                    T.cast(T.Sequence[GPSPoint], points)
                 ),
             )
             if not points:
@@ -181,14 +174,3 @@ class VideoDataExtractor:
             raise exceptions.MapillaryStationaryVideoError("Stationary video")
 
         return points
-
-    @staticmethod
-    def _rebase_times(points: T.Sequence[geo.Point]):
-        """
-        Make point times start from 0
-        """
-        if points:
-            first_timestamp = points[0].time
-            for p in points:
-                p.time = p.time - first_timestamp
-        return points

mapillary_tools/video_data_extraction/extractors/base_parser.py

@@ -2,15 +2,11 @@ import abc
 import functools
 import logging
 import os
-import sys
 import typing as T
 from pathlib import Path
 
-from mapillary_tools import geo
-from mapillary_tools.video_data_extraction.cli_options import (
-    CliOptions,
-    CliParserOptions,
-)
+from ... import geo
+from ..cli_options import CliOptions, CliParserOptions
 
 LOG = logging.getLogger(__name__)
 
@@ -37,11 +33,6 @@ class BaseParser(metaclass=abc.ABCMeta):
     def parser_label(self) -> str:
         raise NotImplementedError
 
-    @property
-    @abc.abstractmethod
-    def must_rebase_times_to_zero(self) -> bool:
-        raise NotImplementedError
-
     @abc.abstractmethod
     def extract_points(self) -> T.Sequence[geo.Point]:
         raise NotImplementedError
@@ -71,3 +62,14 @@ class BaseParser(metaclass=abc.ABCMeta):
         ).resolve()
 
         return abs_path if abs_path.is_file() else None
+
+    @staticmethod
+    def _rebase_times(points: T.Sequence[geo.Point], offset: float = 0.0):
+        """
+        Make point times start from 0
+        """
+        if points:
+            first_timestamp = points[0].time
+            for p in points:
+                p.time = (p.time - first_timestamp) + offset
+        return points

mapillary_tools/video_data_extraction/extractors/blackvue_parser.py

@@ -1,8 +1,9 @@
 import typing as T
 
-from mapillary_tools import geo
-from mapillary_tools.geotag import blackvue_parser, simple_mp4_parser
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo
+from ...geotag import blackvue_parser
+from ...mp4 import simple_mp4_parser as sparser
+from .base_parser import BaseParser
 
 
 class BlackVueParser(BaseParser):
@@ -21,7 +22,7 @@ class BlackVueParser(BaseParser):
                 points = blackvue_parser.extract_points(fp) or []
                 self.pointsFound = len(points) > 0
                 return points
-            except simple_mp4_parser.ParsingError:
+            except sparser.ParsingError:
                 return []
 
     def extract_make(self) -> T.Optional[str]:

mapillary_tools/video_data_extraction/extractors/camm_parser.py

@@ -1,9 +1,10 @@
 import functools
 import typing as T
 
-from mapillary_tools import geo
-from mapillary_tools.geotag import camm_parser, simple_mp4_parser
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo
+from ...camm import camm_parser
+from ...mp4 import simple_mp4_parser as sparser
+from .base_parser import BaseParser
 
 
 class CammParser(BaseParser):
@@ -12,8 +13,12 @@ class CammParser(BaseParser):
     parser_label = "camm"
 
     @functools.cached_property
-    def __camera_info(self) -> T.Tuple[str, str]:
-        with self.videoPath.open("rb") as fp:
+    def _camera_info(self) -> T.Tuple[str, str]:
+        source_path = self.geotag_source_path
+        if not source_path:
+            return "", ""
+
+        with source_path.open("rb") as fp:
             return camm_parser.extract_camera_make_and_model(fp)
 
     def extract_points(self) -> T.Sequence[geo.Point]:
@@ -23,19 +28,11 @@ class CammParser(BaseParser):
         with source_path.open("rb") as fp:
             try:
                 return camm_parser.extract_points(fp) or []
-            except simple_mp4_parser.ParsingError:
+            except sparser.ParsingError:
                 return []
 
     def extract_make(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as fp:
-            return self.__camera_info[0] or None
+        return self._camera_info[0] or None
 
     def extract_model(self) -> T.Optional[str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-        with source_path.open("rb") as fp:
-            return self.__camera_info[1] or None
+        return self._camera_info[1] or None

mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py

@@ -3,15 +3,10 @@ import subprocess
 import typing as T
 from pathlib import Path
 
-from mapillary_tools import constants, exceptions, geo
-from mapillary_tools.video_data_extraction.cli_options import (
-    CliOptions,
-    CliParserOptions,
-)
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
-from mapillary_tools.video_data_extraction.extractors.exiftool_xml_parser import (
-    ExiftoolXmlParser,
-)
+from ... import constants, exceptions, geo
+from ..cli_options import CliOptions, CliParserOptions
+from .base_parser import BaseParser
+from .exiftool_xml_parser import ExiftoolXmlParser
 
 
 class ExiftoolRuntimeParser(BaseParser):

mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py

@@ -3,20 +3,16 @@ import xml.etree.ElementTree as ET
 
 from pathlib import Path
 
-from mapillary_tools import geo
-from mapillary_tools.exiftool_read import EXIFTOOL_NAMESPACES
-from mapillary_tools.exiftool_read_video import ExifToolReadVideo
-from mapillary_tools.geotag.geotag_videos_from_exiftool_video import _DESCRIPTION_TAG
-from mapillary_tools.video_data_extraction.cli_options import (
-    CliOptions,
-    CliParserOptions,
-)
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo
+from ...exiftool_read import EXIFTOOL_NAMESPACES
+from ...exiftool_read_video import ExifToolReadVideo
+from ...geotag.geotag_videos_from_exiftool_video import _DESCRIPTION_TAG
+from ..cli_options import CliOptions, CliParserOptions
+from .base_parser import BaseParser
 
 
 class ExiftoolXmlParser(BaseParser):
     default_source_pattern = "%g.xml"
-    must_rebase_times_to_zero = True
     parser_label = "exiftool_xml"
 
     exifToolReadVideo: T.Optional[ExifToolReadVideo] = None
@@ -42,9 +38,11 @@ class ExiftoolXmlParser(BaseParser):
         self.exifToolReadVideo = ExifToolReadVideo(ET.ElementTree(element))
 
     def extract_points(self) -> T.Sequence[geo.Point]:
-        return (
+        gps_points = (
             self.exifToolReadVideo.extract_gps_track() if self.exifToolReadVideo else []
         )
+        self._rebase_times(gps_points)
+        return gps_points
 
     def extract_make(self) -> T.Optional[str]:
         return self.exifToolReadVideo.extract_make() if self.exifToolReadVideo else None

mapillary_tools/video_data_extraction/extractors/generic_video_parser.py

@@ -1,17 +1,12 @@
 import typing as T
 from pathlib import Path
 
-from mapillary_tools import geo
-from mapillary_tools.video_data_extraction.cli_options import (
-    CliOptions,
-    CliParserOptions,
-)
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
-from mapillary_tools.video_data_extraction.extractors.blackvue_parser import (
-    BlackVueParser,
-)
-from mapillary_tools.video_data_extraction.extractors.camm_parser import CammParser
-from mapillary_tools.video_data_extraction.extractors.gopro_parser import GoProParser
+from ... import geo
+from ..cli_options import CliOptions, CliParserOptions
+from .base_parser import BaseParser
+from .blackvue_parser import BlackVueParser
+from .camm_parser import CammParser
+from .gopro_parser import GoProParser
 
 
 class GenericVideoParser(BaseParser):

mapillary_tools/video_data_extraction/extractors/gopro_parser.py

@@ -1,8 +1,9 @@
 import typing as T
 
-from mapillary_tools import geo
-from mapillary_tools.geotag import gpmf_parser, simple_mp4_parser
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo
+from ...geotag import gpmf_parser
+from ...mp4 import simple_mp4_parser as sparser
+from .base_parser import BaseParser
 
 
 class GoProParser(BaseParser):
@@ -21,10 +22,16 @@ class GoProParser(BaseParser):
                 points = gpmf_parser.extract_points(fp) or []
                 self.pointsFound = len(points) > 0
                 return points
-            except simple_mp4_parser.ParsingError:
+            except sparser.ParsingError:
                 return []
 
     def extract_make(self) -> T.Optional[str]:
+        model = self.extract_model()
+        if model:
+            return "GoPro"
+
+        # make sure self.pointsFound is updated
+        _ = self.extract_points()
         # If no points were found, assume this is not a GoPro
         return "GoPro" if self.pointsFound else None
 

mapillary_tools/video_data_extraction/extractors/gpx_parser.py

@@ -1,29 +1,108 @@
+import datetime
+import logging
 import typing as T
 
-from mapillary_tools import geo
-from mapillary_tools.geotag import geotag_images_from_gpx_file
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo, telemetry
+from ...geotag import geotag_images_from_gpx_file
+from .base_parser import BaseParser
+from .generic_video_parser import GenericVideoParser
+
+
+LOG = logging.getLogger(__name__)
 
 
 class GpxParser(BaseParser):
     default_source_pattern = "%g.gpx"
-    must_rebase_times_to_zero = True
     parser_label = "gpx"
 
     def extract_points(self) -> T.Sequence[geo.Point]:
         path = self.geotag_source_path
         if not path:
             return []
+
         try:
-            tracks = geotag_images_from_gpx_file.parse_gpx(path)
-        except Exception as e:
-            return []
+            gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
+        except Exception as ex:
+            raise RuntimeError(
+                f"Error parsing GPX {path}: {ex.__class__.__name__}: {ex}"
+            )
+
+        if 1 < len(gpx_tracks):
+            LOG.warning(
+                "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
+                len(gpx_tracks),
+                self.videoPath,
+            )
+
+        gpx_points: T.Sequence[geo.Point] = sum(gpx_tracks, [])
+        if not gpx_points:
+            return gpx_points
+
+        offset = self._synx_gpx_by_first_gps_timestamp(gpx_points)
+
+        self._rebase_times(gpx_points, offset=offset)
+
+        return gpx_points
+
+    def _synx_gpx_by_first_gps_timestamp(
+        self, gpx_points: T.Sequence[geo.Point]
+    ) -> float:
+        offset: float = 0.0
+
+        if not gpx_points:
+            return offset
+
+        first_gpx_dt = datetime.datetime.fromtimestamp(
+            gpx_points[0].time, tz=datetime.timezone.utc
+        )
+        LOG.info("First GPX timestamp: %s", first_gpx_dt)
+
+        # Extract first GPS timestamp (if found) for synchronization
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        gps_points = parser.extract_points()
+
+        if not gps_points:
+            LOG.warning(
+                "Skip GPX synchronization because no GPS found in video %s",
+                self.videoPath,
+            )
+            return offset
+
+        first_gps_point = gps_points[0]
+        if isinstance(first_gps_point, telemetry.GPSPoint):
+            if first_gps_point.epoch_time is not None:
+                first_gps_dt = datetime.datetime.fromtimestamp(
+                    first_gps_point.epoch_time, tz=datetime.timezone.utc
+                )
+                LOG.info("First GPS timestamp: %s", first_gps_dt)
+                offset = gpx_points[0].time - first_gps_point.epoch_time
+                if offset:
+                    LOG.warning(
+                        "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
+                        first_gpx_dt,
+                        first_gps_dt,
+                        offset,
+                    )
+                else:
+                    LOG.info(
+                        "GPX and GPS are perfectly synchronized (all starts from %s)",
+                        first_gpx_dt,
+                    )
+            else:
+                LOG.warning(
+                    "Skip GPX synchronization because no GPS epoch time found in video %s",
+                    self.videoPath,
+                )
 
-        points: T.Sequence[geo.Point] = sum(tracks, [])
-        return points
+        return offset
 
     def extract_make(self) -> T.Optional[str]:
-        return None
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        return parser.extract_make()
 
     def extract_model(self) -> T.Optional[str]:
-        return None
+        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
+        parser = GenericVideoParser(self.videoPath, self.options, {})
+        return parser.extract_model()

mapillary_tools/video_data_extraction/extractors/nmea_parser.py

@@ -1,8 +1,8 @@
 import typing as T
 
-from mapillary_tools import geo
-from mapillary_tools.geotag import geotag_images_from_nmea_file
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
+from ... import geo
+from ...geotag import geotag_images_from_nmea_file
+from .base_parser import BaseParser
 
 
 class NmeaParser(BaseParser):

mapillary_tools/video_data_extraction/video_data_parser_factory.py

@@ -1,26 +1,19 @@
 import typing as T
 from pathlib import Path
 
-from mapillary_tools.video_data_extraction.cli_options import CliOptions
-
-from mapillary_tools.video_data_extraction.extractors.base_parser import BaseParser
-
-from mapillary_tools.video_data_extraction.extractors.blackvue_parser import (
-    BlackVueParser,
-)
-from mapillary_tools.video_data_extraction.extractors.camm_parser import CammParser
-from mapillary_tools.video_data_extraction.extractors.exiftool_runtime_parser import (
-    ExiftoolRuntimeParser,
-)
-from mapillary_tools.video_data_extraction.extractors.exiftool_xml_parser import (
-    ExiftoolXmlParser,
-)
-from mapillary_tools.video_data_extraction.extractors.generic_video_parser import (
-    GenericVideoParser,
-)
-from mapillary_tools.video_data_extraction.extractors.gopro_parser import GoProParser
-from mapillary_tools.video_data_extraction.extractors.gpx_parser import GpxParser
-from mapillary_tools.video_data_extraction.extractors.nmea_parser import NmeaParser
+from .cli_options import CliOptions
+
+from .extractors.base_parser import BaseParser
+
+from .extractors.blackvue_parser import BlackVueParser
+from .extractors.camm_parser import CammParser
+
+from .extractors.exiftool_runtime_parser import ExiftoolRuntimeParser
+from .extractors.exiftool_xml_parser import ExiftoolXmlParser
+from .extractors.generic_video_parser import GenericVideoParser
+from .extractors.gopro_parser import GoProParser
+from .extractors.gpx_parser import GpxParser
+from .extractors.nmea_parser import NmeaParser
 
 
 known_parsers = {

{mapillary_tools-0.12.1.dist-info → mapillary_tools-0.13.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mapillary_tools
-Version: 0.12.1
+Version: 0.13.1
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -18,7 +18,14 @@ Requires-Dist: requests[socks]<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing_extensions
 Requires-Dist: jsonschema~=4.17.3
-Requires-Dist: dataclasses; python_version <= "3.6"
+Dynamic: author
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 <p align="center">
   <a href="https://github.com/mapillary/mapillary_tools/">