mapillary-tools 0.14.0a1__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -1,157 +0,0 @@
-from __future__ import annotations
-
-import logging
-import typing as T
-from pathlib import Path
-
-import tqdm
-
-from .. import exceptions, geo, utils
-from ..gpmf import gpmf_gps_filter
-from ..telemetry import GPSPoint
-from ..types import ErrorMetadata, FileType, VideoMetadata, VideoMetadataOrError
-from . import video_data_parser_factory
-from .cli_options import CliOptions
-from .extractors.base_parser import BaseParser
-
-
-LOG = logging.getLogger(__name__)
-
-
-class VideoDataExtractor:
-    options: CliOptions
-
-    def __init__(self, options: CliOptions) -> None:
-        self.options = options
-
-    def process(self) -> T.List[VideoMetadataOrError]:
-        paths = self.options["paths"]
-        self._check_paths(paths)
-        video_files = utils.find_videos(paths)
-        self._check_sources_cardinality(video_files)
-
-        map_results = utils.mp_map_maybe(
-            self.process_file, video_files, num_processes=self.options["num_processes"]
-        )
-
-        video_metadata_or_errors: list[VideoMetadataOrError] = list(
-            tqdm.tqdm(
-                map_results,
-                desc="Extracting GPS tracks",
-                unit="videos",
-                disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                total=len(video_files),
-            )
-        )
-
-        return video_metadata_or_errors
-
-    def process_file(self, file: Path) -> VideoMetadataOrError:
-        parsers = video_data_parser_factory.make_parsers(file, self.options)
-        points: T.Sequence[geo.Point] = []
-        make = self.options["device_make"]
-        model = self.options["device_model"]
-
-        ex: T.Optional[Exception]
-        for parser in parsers:
-            log_vars = {
-                "filename": file,
-                "parser": parser.parser_label,
-                "source": parser.geotag_source_path,
-            }
-            try:
-                if not points:
-                    points = self._extract_points(parser, log_vars)
-                if not model:
-                    model = parser.extract_model()
-                if not make:
-                    make = parser.extract_make()
-            except Exception as e:
-                ex = e
-                LOG.warning(
-                    '%(filename)s: Exception for parser %(parser)s while processing source %(source)s: "%(e)s"',
-                    {**log_vars, "e": e},
-                )
-
-        # After trying all parsers, return the points if we found any, otherwise
-        # the last exception thrown or a default one.
-        # Note that if we have points, we return them, regardless of exceptions
-        # with make or model.
-        if points:
-            video_metadata = VideoMetadata(
-                filename=file,
-                filetype=FileType.VIDEO,
-                filesize=utils.get_file_size(file),
-                points=points,
-                make=make,
-                model=model,
-            )
-            return video_metadata
-        else:
-            return ErrorMetadata(
-                filename=file,
-                error=(
-                    ex
-                    if ex
-                    else exceptions.MapillaryVideoGPSNotFoundError(
-                        "No GPS data found from the video"
-                    )
-                ),
-                filetype=FileType.VIDEO,
-            )
-
-    def _extract_points(
-        self, parser: BaseParser, log_vars: T.Dict
-    ) -> T.Sequence[geo.Point]:
-        points = parser.extract_points()
-        if points:
-            LOG.debug(
-                "%(filename)s: %(points)d points extracted by parser %(parser)s from file %(source)s}",
-                {**log_vars, "points": len(points)},
-            )
-
-        return self._sanitize_points(points)
-
-    @staticmethod
-    def _check_paths(import_paths: T.Sequence[Path]):
-        for path in import_paths:
-            if not path.is_file() and not path.is_dir():
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Import file or directory not found: {path}"
-                )
-
-    def _check_sources_cardinality(self, files: T.Sequence[Path]):
-        if len(files) > 1:
-            for parser_opts in self.options["geotag_sources_options"]:
-                pattern = parser_opts.get("pattern")
-                if pattern and "%" not in pattern:
-                    raise exceptions.MapillaryUserError(
-                        "Multiple video files found: Geotag source pattern for source %s must include filename placeholders",
-                        parser_opts["source"],
-                    )
-
-    @staticmethod
-    def _sanitize_points(points: T.Sequence[geo.Point]) -> T.Sequence[geo.Point]:
-        """
-        Deduplicates points, when possible removes noisy ones, and checks
-        against stationary videos
-        """
-
-        if not points:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found in the given sources"
-            )
-
-        points = geo.extend_deduplicate_points(points)
-
-        if all(isinstance(p, GPSPoint) for p in points):
-            points = T.cast(
-                T.Sequence[geo.Point],
-                gpmf_gps_filter.remove_noisy_points(
-                    T.cast(T.Sequence[GPSPoint], points)
-                ),
-            )
-            if not points:
-                raise exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-
-        return points

mapillary_tools/video_data_extraction/extractors/base_parser.py

@@ -1,75 +0,0 @@
-import abc
-import functools
-import logging
-import os
-import typing as T
-from pathlib import Path
-
-from ... import geo
-from ..cli_options import CliOptions, CliParserOptions
-
-LOG = logging.getLogger(__name__)
-
-
-class BaseParser(metaclass=abc.ABCMeta):
-    videoPath: Path
-    options: CliOptions
-    parserOptions: CliParserOptions
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ) -> None:
-        self.videoPath = video_path
-        self.options = options
-        self.parserOptions = parser_options
-
-    @property
-    @abc.abstractmethod
-    def default_source_pattern(self) -> str:
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def parser_label(self) -> str:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_make(self) -> T.Optional[str]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_model(self) -> T.Optional[str]:
-        raise NotImplementedError
-
-    @functools.cached_property
-    def geotag_source_path(self) -> T.Optional[Path]:
-        video_dir = self.videoPath.parent.resolve()
-        video_filename = self.videoPath.name
-        video_basename, video_ext = os.path.splitext(video_filename)
-        pattern = self.parserOptions.get("pattern") or self.default_source_pattern
-
-        replaced = Path(
-            pattern.replace("%f", video_filename)
-            .replace("%g", video_basename)
-            .replace("%e", video_ext)
-        )
-        abs_path = (
-            replaced if replaced.is_absolute() else Path.joinpath(video_dir, replaced)
-        ).resolve()
-
-        return abs_path if abs_path.is_file() else None
-
-    @staticmethod
-    def _rebase_times(points: T.Sequence[geo.Point], offset: float = 0.0):
-        """
-        Make point times start from 0
-        """
-        if points:
-            first_timestamp = points[0].time
-            for p in points:
-                p.time = (p.time - first_timestamp) + offset
-        return points

mapillary_tools/video_data_extraction/extractors/blackvue_parser.py

@@ -1,49 +0,0 @@
-from __future__ import annotations
-
-import functools
-
-import typing as T
-
-from ... import blackvue_parser, geo
-from .base_parser import BaseParser
-
-
-class BlackVueParser(BaseParser):
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "blackvue"
-
-    pointsFound: bool = False
-
-    @functools.cached_property
-    def extract_blackvue_info(self) -> blackvue_parser.BlackVueInfo | None:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return None
-
-        with source_path.open("rb") as fp:
-            return blackvue_parser.extract_blackvue_info(fp)
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        blackvue_info = self.extract_blackvue_info
-
-        if blackvue_info is None:
-            return []
-
-        return blackvue_info.gps or []
-
-    def extract_make(self) -> str | None:
-        blackvue_info = self.extract_blackvue_info
-
-        if blackvue_info is None:
-            return None
-
-        return blackvue_info.make
-
-    def extract_model(self) -> str | None:
-        blackvue_info = self.extract_blackvue_info
-
-        if blackvue_info is None:
-            return None
-
-        return blackvue_info.model

mapillary_tools/video_data_extraction/extractors/camm_parser.py

@@ -1,62 +0,0 @@
-from __future__ import annotations
-
-import typing as T
-
-from ... import geo
-from ...camm import camm_parser
-from ...mp4 import simple_mp4_parser as sparser
-from .base_parser import BaseParser
-
-
-class CammParser(BaseParser):
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "camm"
-
-    _extracted: bool = False
-    _cached_camm_info: camm_parser.CAMMInfo | None = None
-
-    # TODO: use @functools.cached_property
-    def _extract_camm_info(self) -> camm_parser.CAMMInfo | None:
-        if self._extracted:
-            return self._cached_camm_info
-
-        self._extracted = True
-
-        source_path = self.geotag_source_path
-
-        if source_path is None:
-            # source_path not found
-            return None
-
-        with source_path.open("rb") as fp:
-            try:
-                self._cached_camm_info = camm_parser.extract_camm_info(fp)
-            except sparser.ParsingError:
-                self._cached_camm_info = None
-
-        return self._cached_camm_info
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        camm_info = self._extract_camm_info()
-
-        if camm_info is None:
-            return []
-
-        return T.cast(T.List[geo.Point], camm_info.gps or camm_info.mini_gps)
-
-    def extract_make(self) -> str | None:
-        camm_info = self._extract_camm_info()
-
-        if camm_info is None:
-            return None
-
-        return camm_info.make
-
-    def extract_model(self) -> str | None:
-        camm_info = self._extract_camm_info()
-
-        if camm_info is None:
-            return None
-
-        return camm_info.model

mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py

@@ -1,74 +0,0 @@
-import shutil
-import subprocess
-import typing as T
-from pathlib import Path
-
-from ... import constants, exceptions, geo
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-from .exiftool_xml_parser import ExiftoolXmlParser
-
-
-class ExiftoolRuntimeParser(BaseParser):
-    """
-    Wrapper around ExiftoolRdfParser that executes exiftool
-    """
-
-    exiftoolXmlParser: ExiftoolXmlParser
-
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = True
-    parser_label = "exiftool_runtime"
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ):
-        super().__init__(video_path, options, parser_options)
-        if constants.EXIFTOOL_PATH is None:
-            exiftool_path = shutil.which("exiftool")
-        else:
-            exiftool_path = shutil.which(constants.EXIFTOOL_PATH)
-
-        if not exiftool_path:
-            raise exceptions.MapillaryExiftoolNotFoundError(
-                "Cannot execute exiftool. Please install it from https://exiftool.org/ or you package manager, or set the environment variable MAPILLARY_TOOLS_EXIFTOOL_PATH"
-            )
-        if not self.geotag_source_path:
-            return
-
-        # To handle non-latin1 filenames under Windows, we pass the path
-        # via stdin. See https://exiftool.org/faq.html#Q18
-        stdin = str(self.geotag_source_path)
-        args = [
-            exiftool_path,
-            "-q",
-            "-r",
-            "-n",
-            "-ee",
-            "-api",
-            "LargeFileSupport=1",
-            "-X",
-            "-charset",
-            "filename=utf8",
-            "-@",
-            "-",
-        ]
-
-        process = subprocess.run(
-            args, capture_output=True, text=True, input=stdin, encoding="utf-8"
-        )
-
-        self.exiftoolXmlParser = ExiftoolXmlParser(
-            video_path, options, parser_options, process.stdout
-        )
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        return self.exiftoolXmlParser.extract_points() if self.exiftoolXmlParser else []
-
-    def extract_make(self) -> T.Optional[str]:
-        return self.exiftoolXmlParser.extract_make() if self.exiftoolXmlParser else None
-
-    def extract_model(self) -> T.Optional[str]:
-        return (
-            self.exiftoolXmlParser.extract_model() if self.exiftoolXmlParser else None
-        )

mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py

@@ -1,52 +0,0 @@
-import typing as T
-import xml.etree.ElementTree as ET
-
-from pathlib import Path
-
-from ... import geo
-from ...exiftool_read import _DESCRIPTION_TAG, EXIFTOOL_NAMESPACES
-from ...exiftool_read_video import ExifToolReadVideo
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-
-
-class ExiftoolXmlParser(BaseParser):
-    default_source_pattern = "%g.xml"
-    parser_label = "exiftool_xml"
-
-    exifToolReadVideo: T.Optional[ExifToolReadVideo] = None
-
-    def __init__(
-        self,
-        video_path: Path,
-        options: CliOptions,
-        parser_options: CliParserOptions,
-        xml_content: T.Optional[str] = None,
-    ) -> None:
-        super().__init__(video_path, options, parser_options)
-
-        if xml_content:
-            etree = ET.fromstring(xml_content)
-        else:
-            xml_path = self.geotag_source_path
-            if not xml_path:
-                return
-            etree = ET.parse(xml_path).getroot()
-
-        element = next(etree.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES))
-        self.exifToolReadVideo = ExifToolReadVideo(ET.ElementTree(element))
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        gps_points = (
-            self.exifToolReadVideo.extract_gps_track() if self.exifToolReadVideo else []
-        )
-        self._rebase_times(gps_points)
-        return gps_points
-
-    def extract_make(self) -> T.Optional[str]:
-        return self.exifToolReadVideo.extract_make() if self.exifToolReadVideo else None
-
-    def extract_model(self) -> T.Optional[str]:
-        return (
-            self.exifToolReadVideo.extract_model() if self.exifToolReadVideo else None
-        )

mapillary_tools/video_data_extraction/extractors/generic_video_parser.py

@@ -1,52 +0,0 @@
-import typing as T
-from pathlib import Path
-
-from ... import geo
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-from .blackvue_parser import BlackVueParser
-from .camm_parser import CammParser
-from .gopro_parser import GoProParser
-
-
-class GenericVideoParser(BaseParser):
-    """
-    Wrapper around the three native video parsers. It will try to execute them
-    in the order camm-gopro-blackvue, like the previous implementation
-    """
-
-    parsers: T.Sequence[BaseParser] = []
-
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "video"
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ) -> None:
-        super().__init__(video_path, options, parser_options)
-        camm_parser = CammParser(video_path, options, parser_options)
-        gopro_parser = GoProParser(video_path, options, parser_options)
-        blackvue_parser = BlackVueParser(video_path, options, parser_options)
-        self.parsers = [camm_parser, gopro_parser, blackvue_parser]
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        for parser in self.parsers:
-            points = parser.extract_points()
-            if points:
-                return points
-        return []
-
-    def extract_make(self) -> T.Optional[str]:
-        for parser in self.parsers:
-            make = parser.extract_make()
-            if make:
-                return make
-        return None
-
-    def extract_model(self) -> T.Optional[str]:
-        for parser in self.parsers:
-            model = parser.extract_model()
-            if model:
-                return model
-        return None

mapillary_tools/video_data_extraction/extractors/gopro_parser.py

@@ -1,58 +0,0 @@
-from __future__ import annotations
-
-import typing as T
-
-from ... import geo
-from ...gpmf import gpmf_parser
-from ...mp4 import simple_mp4_parser as sparser
-from .base_parser import BaseParser
-
-
-class GoProParser(BaseParser):
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "gopro"
-
-    _extracted: bool = False
-    _cached_gopro_info: gpmf_parser.GoProInfo | None = None
-
-    def _extract_gopro_info(self) -> gpmf_parser.GoProInfo | None:
-        if self._extracted:
-            return self._cached_gopro_info
-
-        self._extracted = True
-
-        source_path = self.geotag_source_path
-
-        if source_path is None:
-            # source_path not found
-            return None
-
-        with source_path.open("rb") as fp:
-            try:
-                self._cached_gopro_info = gpmf_parser.extract_gopro_info(fp)
-            except sparser.ParsingError:
-                self._cached_gopro_info = None
-
-        return self._cached_gopro_info
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        gopro_info = self._extract_gopro_info()
-        if gopro_info is None:
-            return []
-
-        return T.cast(T.Sequence[geo.Point], gopro_info.gps)
-
-    def extract_make(self) -> str | None:
-        gopro_info = self._extract_gopro_info()
-        if gopro_info is None:
-            return None
-
-        return gopro_info.make
-
-    def extract_model(self) -> str | None:
-        gopro_info = self._extract_gopro_info()
-        if gopro_info is None:
-            return None
-
-        return gopro_info.model

mapillary_tools/video_data_extraction/extractors/gpx_parser.py

@@ -1,108 +0,0 @@
-import datetime
-import logging
-import typing as T
-
-from ... import geo, telemetry
-from ...geotag import geotag_images_from_gpx_file
-from .base_parser import BaseParser
-from .generic_video_parser import GenericVideoParser
-
-
-LOG = logging.getLogger(__name__)
-
-
-class GpxParser(BaseParser):
-    default_source_pattern = "%g.gpx"
-    parser_label = "gpx"
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        path = self.geotag_source_path
-        if not path:
-            return []
-
-        try:
-            gpx_tracks = geotag_images_from_gpx_file.parse_gpx(path)
-        except Exception as ex:
-            raise RuntimeError(
-                f"Error parsing GPX {path}: {ex.__class__.__name__}: {ex}"
-            )
-
-        if 1 < len(gpx_tracks):
-            LOG.warning(
-                "Found %s tracks in the GPX file %s. Will merge points in all the tracks as a single track for interpolation",
-                len(gpx_tracks),
-                self.videoPath,
-            )
-
-        gpx_points: T.Sequence[geo.Point] = sum(gpx_tracks, [])
-        if not gpx_points:
-            return gpx_points
-
-        offset = self._synx_gpx_by_first_gps_timestamp(gpx_points)
-
-        self._rebase_times(gpx_points, offset=offset)
-
-        return gpx_points
-
-    def _synx_gpx_by_first_gps_timestamp(
-        self, gpx_points: T.Sequence[geo.Point]
-    ) -> float:
-        offset: float = 0.0
-
-        if not gpx_points:
-            return offset
-
-        first_gpx_dt = datetime.datetime.fromtimestamp(
-            gpx_points[0].time, tz=datetime.timezone.utc
-        )
-        LOG.info("First GPX timestamp: %s", first_gpx_dt)
-
-        # Extract first GPS timestamp (if found) for synchronization
-        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
-        parser = GenericVideoParser(self.videoPath, self.options, {})
-        gps_points = parser.extract_points()
-
-        if not gps_points:
-            LOG.warning(
-                "Skip GPX synchronization because no GPS found in video %s",
-                self.videoPath,
-            )
-            return offset
-
-        first_gps_point = gps_points[0]
-        if isinstance(first_gps_point, telemetry.GPSPoint):
-            if first_gps_point.epoch_time is not None:
-                first_gps_dt = datetime.datetime.fromtimestamp(
-                    first_gps_point.epoch_time, tz=datetime.timezone.utc
-                )
-                LOG.info("First GPS timestamp: %s", first_gps_dt)
-                offset = gpx_points[0].time - first_gps_point.epoch_time
-                if offset:
-                    LOG.warning(
-                        "Found offset between GPX %s and video GPS timestamps %s: %s seconds",
-                        first_gpx_dt,
-                        first_gps_dt,
-                        offset,
-                    )
-                else:
-                    LOG.info(
-                        "GPX and GPS are perfectly synchronized (all starts from %s)",
-                        first_gpx_dt,
-                    )
-            else:
-                LOG.warning(
-                    "Skip GPX synchronization because no GPS epoch time found in video %s",
-                    self.videoPath,
-                )
-
-        return offset
-
-    def extract_make(self) -> T.Optional[str]:
-        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
-        parser = GenericVideoParser(self.videoPath, self.options, {})
-        return parser.extract_make()
-
-    def extract_model(self) -> T.Optional[str]:
-        # Use an empty dictionary to force video parsers to extract make/model from the video metadata itself
-        parser = GenericVideoParser(self.videoPath, self.options, {})
-        return parser.extract_model()