mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/geotag/geotag_videos_from_exiftool_video.py

@@ -1,145 +0,0 @@
-import logging
-import typing as T
-import xml.etree.ElementTree as ET
-from multiprocessing import Pool
-from pathlib import Path
-
-from tqdm import tqdm
-
-from .. import exceptions, exiftool_read, geo, types, utils
-from ..exiftool_read_video import ExifToolReadVideo
-from ..telemetry import GPSPoint
-from . import gpmf_gps_filter, utils as video_utils
-from .geotag_from_generic import GeotagVideosFromGeneric
-
-LOG = logging.getLogger(__name__)
-_DESCRIPTION_TAG = "rdf:Description"
-
-
-class GeotagVideosFromExifToolVideo(GeotagVideosFromGeneric):
-    def __init__(
-        self,
-        video_paths: T.Sequence[Path],
-        xml_path: Path,
-        num_processes: T.Optional[int] = None,
-    ):
-        self.video_paths = video_paths
-        self.xml_path = xml_path
-        self.num_processes = num_processes
-        super().__init__()
-
-    @staticmethod
-    def geotag_video(element: ET.Element) -> types.VideoMetadataOrError:
-        video_path = exiftool_read.find_rdf_description_path(element)
-        assert video_path is not None, "must find the path from the element"
-
-        try:
-            exif = ExifToolReadVideo(ET.ElementTree(element))
-
-            points = exif.extract_gps_track()
-
-            if not points:
-                raise exceptions.MapillaryVideoGPSNotFoundError(
-                    "No GPS data found from the video"
-                )
-
-            points = geo.extend_deduplicate_points(points)
-            assert points, "must have at least one point"
-
-            if all(isinstance(p, GPSPoint) for p in points):
-                points = T.cast(
-                    T.List[geo.Point],
-                    gpmf_gps_filter.remove_noisy_points(
-                        T.cast(T.List[GPSPoint], points)
-                    ),
-                )
-                if not points:
-                    raise exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-
-            stationary = video_utils.is_video_stationary(
-                geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
-            )
-
-            if stationary:
-                raise exceptions.MapillaryStationaryVideoError("Stationary video")
-
-            video_metadata = types.VideoMetadata(
-                video_path,
-                md5sum=None,
-                filesize=utils.get_file_size(video_path),
-                filetype=types.FileType.VIDEO,
-                points=points,
-                make=exif.extract_make(),
-                model=exif.extract_model(),
-            )
-
-            LOG.debug("Calculating MD5 checksum for %s", str(video_metadata.filename))
-
-            video_metadata.update_md5sum()
-
-        except Exception as ex:
-            if not isinstance(ex, exceptions.MapillaryDescriptionError):
-                LOG.warning(
-                    "Failed to geotag video %s: %s",
-                    video_path,
-                    str(ex),
-                    exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
-                )
-            return types.describe_error_metadata(
-                ex, video_path, filetype=types.FileType.VIDEO
-            )
-
-        return video_metadata
-
-    def to_description(self) -> T.List[types.VideoMetadataOrError]:
-        rdf_description_by_path = exiftool_read.index_rdf_description_by_path(
-            [self.xml_path]
-        )
-
-        error_metadatas: T.List[types.ErrorMetadata] = []
-        rdf_descriptions: T.List[ET.Element] = []
-        for path in self.video_paths:
-            rdf_description = rdf_description_by_path.get(
-                exiftool_read.canonical_path(path)
-            )
-            if rdf_description is None:
-                exc = exceptions.MapillaryEXIFNotFoundError(
-                    f"The {_DESCRIPTION_TAG} XML element for the video not found"
-                )
-                error_metadatas.append(
-                    types.describe_error_metadata(
-                        exc, path, filetype=types.FileType.VIDEO
-                    )
-                )
-            else:
-                rdf_descriptions.append(rdf_description)
-
-        if self.num_processes is None:
-            num_processes = self.num_processes
-            disable_multiprocessing = False
-        else:
-            num_processes = max(self.num_processes, 1)
-            disable_multiprocessing = self.num_processes <= 0
-
-        with Pool(processes=num_processes) as pool:
-            video_metadatas_iter: T.Iterator[types.VideoMetadataOrError]
-            if disable_multiprocessing:
-                video_metadatas_iter = map(
-                    GeotagVideosFromExifToolVideo.geotag_video, rdf_descriptions
-                )
-            else:
-                video_metadatas_iter = pool.imap(
-                    GeotagVideosFromExifToolVideo.geotag_video,
-                    rdf_descriptions,
-                )
-            video_metadata_or_errors = list(
-                tqdm(
-                    video_metadatas_iter,
-                    desc="Extracting GPS tracks from ExifTool XML",
-                    unit="videos",
-                    disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                    total=len(self.video_paths),
-                )
-            )
-
-        return error_metadatas + video_metadata_or_errors

mapillary_tools/video_data_extraction/cli_options.py

@@ -1,22 +0,0 @@
-import typing as T
-from pathlib import Path
-
-
-known_parser_options = ["source", "pattern", "exiftool_path"]
-
-
-class CliParserOptions(T.TypedDict, total=False):
-    source: str
-    pattern: T.Optional[str]
-    exiftool_path: T.Optional[Path]
-
-
-class CliOptions(T.TypedDict, total=False):
-    paths: T.Sequence[Path]
-    recursive: bool
-    geotag_sources_options: T.Sequence[CliParserOptions]
-    geotag_source_path: Path
-    exiftool_path: Path
-    num_processes: int
-    device_make: T.Optional[str]
-    device_model: T.Optional[str]

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -1,176 +0,0 @@
-import logging
-import typing as T
-from multiprocessing import Pool
-from pathlib import Path
-
-import tqdm
-
-from .. import exceptions, geo, utils
-from ..geotag import gpmf_gps_filter, utils as video_utils
-from ..telemetry import GPSPoint
-from ..types import (
-    ErrorMetadata,
-    FileType,
-    MetadataOrError,
-    VideoMetadata,
-    VideoMetadataOrError,
-)
-from . import video_data_parser_factory
-from .cli_options import CliOptions
-from .extractors.base_parser import BaseParser
-
-
-LOG = logging.getLogger(__name__)
-
-
-class VideoDataExtractor:
-    options: CliOptions
-
-    def __init__(self, options: CliOptions) -> None:
-        self.options = options
-
-    def process(self) -> T.List[MetadataOrError]:
-        paths = self.options["paths"]
-        self._check_paths(paths)
-        video_files = utils.find_videos(paths)
-        self._check_sources_cardinality(video_files)
-
-        num_processes = self.options["num_processes"] or None
-        with Pool(processes=num_processes) as pool:
-            if num_processes == 1:
-                iter: T.Iterator[VideoMetadataOrError] = map(
-                    self.process_file, video_files
-                )
-            else:
-                iter = pool.imap(self.process_file, video_files)
-
-            video_metadata_or_errors = list(
-                tqdm.tqdm(
-                    iter,
-                    desc="Extracting GPS tracks",
-                    unit="videos",
-                    disable=LOG.getEffectiveLevel() <= logging.DEBUG,
-                    total=len(video_files),
-                )
-            )
-
-        return video_metadata_or_errors
-
-    def process_file(self, file: Path) -> VideoMetadataOrError:
-        parsers = video_data_parser_factory.make_parsers(file, self.options)
-        points: T.Sequence[geo.Point] = []
-        make = self.options["device_make"]
-        model = self.options["device_model"]
-
-        ex: T.Optional[Exception]
-        for parser in parsers:
-            log_vars = {
-                "filename": file,
-                "parser": parser.parser_label,
-                "source": parser.geotag_source_path,
-            }
-            try:
-                if not points:
-                    points = self._extract_points(parser, log_vars)
-                if not model:
-                    model = parser.extract_model()
-                if not make:
-                    make = parser.extract_make()
-            except Exception as e:
-                ex = e
-                LOG.warning(
-                    '%(filename)s: Exception for parser %(parser)s while processing source %(source)s: "%(e)s"',
-                    {**log_vars, "e": e},
-                )
-
-        # After trying all parsers, return the points if we found any, otherwise
-        # the last exception thrown or a default one.
-        # Note that if we have points, we return them, regardless of exceptions
-        # with make or model.
-        if points:
-            video_metadata = VideoMetadata(
-                filename=file,
-                filetype=FileType.VIDEO,
-                md5sum=None,
-                filesize=utils.get_file_size(file),
-                points=points,
-                make=make,
-                model=model,
-            )
-            video_metadata.update_md5sum()
-            return video_metadata
-        else:
-            return ErrorMetadata(
-                filename=file,
-                error=(
-                    ex
-                    if ex
-                    else exceptions.MapillaryVideoGPSNotFoundError(
-                        "No GPS data found from the video"
-                    )
-                ),
-                filetype=FileType.VIDEO,
-            )
-
-    def _extract_points(
-        self, parser: BaseParser, log_vars: T.Dict
-    ) -> T.Sequence[geo.Point]:
-        points = parser.extract_points()
-        if points:
-            LOG.debug(
-                "%(filename)s: %(points)d points extracted by parser %(parser)s from file %(source)s}",
-                {**log_vars, "points": len(points)},
-            )
-
-        return self._sanitize_points(points)
-
-    @staticmethod
-    def _check_paths(import_paths: T.Sequence[Path]):
-        for path in import_paths:
-            if not path.is_file() and not path.is_dir():
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Import file or directory not found: {path}"
-                )
-
-    def _check_sources_cardinality(self, files: T.Sequence[Path]):
-        if len(files) > 1:
-            for parser_opts in self.options["geotag_sources_options"]:
-                pattern = parser_opts.get("pattern")
-                if pattern and "%" not in pattern:
-                    raise exceptions.MapillaryUserError(
-                        "Multiple video files found: Geotag source pattern for source %s must include filename placeholders",
-                        parser_opts["source"],
-                    )
-
-    @staticmethod
-    def _sanitize_points(points: T.Sequence[geo.Point]) -> T.Sequence[geo.Point]:
-        """
-        Deduplicates points, when possible removes noisy ones, and checks
-        against stationary videos
-        """
-
-        if not points:
-            raise exceptions.MapillaryVideoGPSNotFoundError(
-                "No GPS data found in the given sources"
-            )
-
-        points = geo.extend_deduplicate_points(points)
-
-        if all(isinstance(p, GPSPoint) for p in points):
-            points = T.cast(
-                T.Sequence[geo.Point],
-                gpmf_gps_filter.remove_noisy_points(
-                    T.cast(T.Sequence[GPSPoint], points)
-                ),
-            )
-            if not points:
-                raise exceptions.MapillaryGPSNoiseError("GPS is too noisy")
-
-        stationary = video_utils.is_video_stationary(
-            geo.get_max_distance_from_start([(p.lat, p.lon) for p in points])
-        )
-
-        if stationary:
-            raise exceptions.MapillaryStationaryVideoError("Stationary video")
-
-        return points

mapillary_tools/video_data_extraction/extractors/base_parser.py

@@ -1,75 +0,0 @@
-import abc
-import functools
-import logging
-import os
-import typing as T
-from pathlib import Path
-
-from ... import geo
-from ..cli_options import CliOptions, CliParserOptions
-
-LOG = logging.getLogger(__name__)
-
-
-class BaseParser(metaclass=abc.ABCMeta):
-    videoPath: Path
-    options: CliOptions
-    parserOptions: CliParserOptions
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ) -> None:
-        self.videoPath = video_path
-        self.options = options
-        self.parserOptions = parser_options
-
-    @property
-    @abc.abstractmethod
-    def default_source_pattern(self) -> str:
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def parser_label(self) -> str:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_make(self) -> T.Optional[str]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def extract_model(self) -> T.Optional[str]:
-        raise NotImplementedError
-
-    @functools.cached_property
-    def geotag_source_path(self) -> T.Optional[Path]:
-        video_dir = self.videoPath.parent.resolve()
-        video_filename = self.videoPath.name
-        video_basename, video_ext = os.path.splitext(video_filename)
-        pattern = self.parserOptions.get("pattern") or self.default_source_pattern
-
-        replaced = Path(
-            pattern.replace("%f", video_filename)
-            .replace("%g", video_basename)
-            .replace("%e", video_ext)
-        )
-        abs_path = (
-            replaced if replaced.is_absolute() else Path.joinpath(video_dir, replaced)
-        ).resolve()
-
-        return abs_path if abs_path.is_file() else None
-
-    @staticmethod
-    def _rebase_times(points: T.Sequence[geo.Point], offset: float = 0.0):
-        """
-        Make point times start from 0
-        """
-        if points:
-            first_timestamp = points[0].time
-            for p in points:
-                p.time = (p.time - first_timestamp) + offset
-        return points

mapillary_tools/video_data_extraction/extractors/blackvue_parser.py

@@ -1,34 +0,0 @@
-import typing as T
-
-from ... import geo
-from ...geotag import blackvue_parser
-from ...mp4 import simple_mp4_parser as sparser
-from .base_parser import BaseParser
-
-
-class BlackVueParser(BaseParser):
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "blackvue"
-
-    pointsFound: bool = False
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return []
-        with source_path.open("rb") as fp:
-            try:
-                points = blackvue_parser.extract_points(fp) or []
-                self.pointsFound = len(points) > 0
-                return points
-            except sparser.ParsingError:
-                return []
-
-    def extract_make(self) -> T.Optional[str]:
-        # If no points were found, assume this is not a BlackVue
-        return "Blackvue" if self.pointsFound else None
-
-    def extract_model(self) -> T.Optional[str]:
-        with self.videoPath.open("rb") as fp:
-            return blackvue_parser.extract_camera_model(fp) or None

mapillary_tools/video_data_extraction/extractors/camm_parser.py

@@ -1,38 +0,0 @@
-import functools
-import typing as T
-
-from ... import geo
-from ...camm import camm_parser
-from ...mp4 import simple_mp4_parser as sparser
-from .base_parser import BaseParser
-
-
-class CammParser(BaseParser):
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "camm"
-
-    @functools.cached_property
-    def _camera_info(self) -> T.Tuple[str, str]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return "", ""
-
-        with source_path.open("rb") as fp:
-            return camm_parser.extract_camera_make_and_model(fp)
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        source_path = self.geotag_source_path
-        if not source_path:
-            return []
-        with source_path.open("rb") as fp:
-            try:
-                return camm_parser.extract_points(fp) or []
-            except sparser.ParsingError:
-                return []
-
-    def extract_make(self) -> T.Optional[str]:
-        return self._camera_info[0] or None
-
-    def extract_model(self) -> T.Optional[str]:
-        return self._camera_info[1] or None

mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py

@@ -1,71 +0,0 @@
-import shutil
-import subprocess
-import typing as T
-from pathlib import Path
-
-from ... import constants, exceptions, geo
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-from .exiftool_xml_parser import ExiftoolXmlParser
-
-
-class ExiftoolRuntimeParser(BaseParser):
-    """
-    Wrapper around ExiftoolRdfParser that executes exiftool
-    """
-
-    exiftoolXmlParser: ExiftoolXmlParser
-
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = True
-    parser_label = "exiftool_runtime"
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ):
-        super().__init__(video_path, options, parser_options)
-        exiftool_path = shutil.which(constants.EXIFTOOL_PATH)
-
-        if not exiftool_path:
-            raise exceptions.MapillaryExiftoolNotFoundError(
-                "Cannot execute exiftool. Please install it from https://exiftool.org/ or you package manager, or set the environment variable MAPILLARY_TOOLS_EXIFTOOL_PATH"
-            )
-        if not self.geotag_source_path:
-            return
-
-        # To handle non-latin1 filenames under Windows, we pass the path
-        # via stdin. See https://exiftool.org/faq.html#Q18
-        stdin = str(self.geotag_source_path)
-        args = [
-            exiftool_path,
-            "-q",
-            "-r",
-            "-n",
-            "-ee",
-            "-api",
-            "LargeFileSupport=1",
-            "-X",
-            "-charset",
-            "filename=utf8",
-            "-@",
-            "-",
-        ]
-
-        process = subprocess.run(
-            args, capture_output=True, text=True, input=stdin, encoding="utf-8"
-        )
-
-        self.exiftoolXmlParser = ExiftoolXmlParser(
-            video_path, options, parser_options, process.stdout
-        )
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        return self.exiftoolXmlParser.extract_points() if self.exiftoolXmlParser else []
-
-    def extract_make(self) -> T.Optional[str]:
-        return self.exiftoolXmlParser.extract_make() if self.exiftoolXmlParser else None
-
-    def extract_model(self) -> T.Optional[str]:
-        return (
-            self.exiftoolXmlParser.extract_model() if self.exiftoolXmlParser else None
-        )

mapillary_tools/video_data_extraction/extractors/exiftool_xml_parser.py

@@ -1,53 +0,0 @@
-import typing as T
-import xml.etree.ElementTree as ET
-
-from pathlib import Path
-
-from ... import geo
-from ...exiftool_read import EXIFTOOL_NAMESPACES
-from ...exiftool_read_video import ExifToolReadVideo
-from ...geotag.geotag_videos_from_exiftool_video import _DESCRIPTION_TAG
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-
-
-class ExiftoolXmlParser(BaseParser):
-    default_source_pattern = "%g.xml"
-    parser_label = "exiftool_xml"
-
-    exifToolReadVideo: T.Optional[ExifToolReadVideo] = None
-
-    def __init__(
-        self,
-        video_path: Path,
-        options: CliOptions,
-        parser_options: CliParserOptions,
-        xml_content: T.Optional[str] = None,
-    ) -> None:
-        super().__init__(video_path, options, parser_options)
-
-        if xml_content:
-            etree = ET.fromstring(xml_content)
-        else:
-            xml_path = self.geotag_source_path
-            if not xml_path:
-                return
-            etree = ET.parse(xml_path).getroot()
-
-        element = next(etree.iterfind(_DESCRIPTION_TAG, namespaces=EXIFTOOL_NAMESPACES))
-        self.exifToolReadVideo = ExifToolReadVideo(ET.ElementTree(element))
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        gps_points = (
-            self.exifToolReadVideo.extract_gps_track() if self.exifToolReadVideo else []
-        )
-        self._rebase_times(gps_points)
-        return gps_points
-
-    def extract_make(self) -> T.Optional[str]:
-        return self.exifToolReadVideo.extract_make() if self.exifToolReadVideo else None
-
-    def extract_model(self) -> T.Optional[str]:
-        return (
-            self.exifToolReadVideo.extract_model() if self.exifToolReadVideo else None
-        )

mapillary_tools/video_data_extraction/extractors/generic_video_parser.py

@@ -1,52 +0,0 @@
-import typing as T
-from pathlib import Path
-
-from ... import geo
-from ..cli_options import CliOptions, CliParserOptions
-from .base_parser import BaseParser
-from .blackvue_parser import BlackVueParser
-from .camm_parser import CammParser
-from .gopro_parser import GoProParser
-
-
-class GenericVideoParser(BaseParser):
-    """
-    Wrapper around the three native video parsers. It will try to execute them
-    in the order camm-gopro-blackvue, like the previous implementation
-    """
-
-    parsers: T.Sequence[BaseParser] = []
-
-    default_source_pattern = "%f"
-    must_rebase_times_to_zero = False
-    parser_label = "video"
-
-    def __init__(
-        self, video_path: Path, options: CliOptions, parser_options: CliParserOptions
-    ) -> None:
-        super().__init__(video_path, options, parser_options)
-        camm_parser = CammParser(video_path, options, parser_options)
-        gopro_parser = GoProParser(video_path, options, parser_options)
-        blackvue_parser = BlackVueParser(video_path, options, parser_options)
-        self.parsers = [camm_parser, gopro_parser, blackvue_parser]
-
-    def extract_points(self) -> T.Sequence[geo.Point]:
-        for parser in self.parsers:
-            points = parser.extract_points()
-            if points:
-                return points
-        return []
-
-    def extract_make(self) -> T.Optional[str]:
-        for parser in self.parsers:
-            make = parser.extract_make()
-            if make:
-                return make
-        return None
-
-    def extract_model(self) -> T.Optional[str]:
-        for parser in self.parsers:
-            model = parser.extract_model()
-            if model:
-                return model
-        return None