mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/history.py

@@ -1,22 +1,29 @@
+from __future__ import annotations
+
+import contextlib
+import dbm
 import json
 import logging
-import os
 import string
+import threading
+import time
 import typing as T
 from pathlib import Path
 
+# dbm modules are dynamically imported, so here we explicitly import dbm.sqlite3 to make sure pyinstaller include it
+# Otherwise you will see: ImportError: no dbm clone found; tried ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
+try:
+    import dbm.sqlite3  # type: ignore
+except ImportError:
+    pass
+
+
 from . import constants, types
+from .serializer.description import DescriptionJSONSerializer
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
 
 LOG = logging.getLogger(__name__)
-MAPILLARY_UPLOAD_HISTORY_PATH = os.getenv(
-    "MAPILLARY_UPLOAD_HISTORY_PATH",
-    os.path.join(
-        constants.USER_DATA_DIR,
-        "upload_history",
-    ),
-)
 
 
 def _validate_hexdigits(md5sum: str):
@@ -35,34 +42,141 @@ def history_desc_path(md5sum: str) -> Path:
     basename = md5sum[2:]
     assert basename, f"Invalid md5sum {md5sum}"
     return (
-        Path(MAPILLARY_UPLOAD_HISTORY_PATH)
+        Path(constants.MAPILLARY_UPLOAD_HISTORY_PATH)
         .joinpath(subfolder)
         .joinpath(f"{basename}.json")
     )
 
 
-def is_uploaded(md5sum: str) -> bool:
-    if not MAPILLARY_UPLOAD_HISTORY_PATH:
-        return False
-    return history_desc_path(md5sum).is_file()
+def read_history_record(md5sum: str) -> None | T.Dict[str, T.Any]:
+    if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
+        return None
+
+    path = history_desc_path(md5sum)
+
+    if not path.is_file():
+        return None
+
+    with path.open("r") as fp:
+        try:
+            return json.load(fp)
+        except json.JSONDecodeError as ex:
+            LOG.error(f"Failed to read upload history {path}: {ex}")
+            return None
 
 
 def write_history(
     md5sum: str,
     params: JSONDict,
     summary: JSONDict,
-    metadatas: T.Optional[T.Sequence[types.Metadata]] = None,
+    metadatas: T.Sequence[types.Metadata] | None = None,
 ) -> None:
-    if not MAPILLARY_UPLOAD_HISTORY_PATH:
+    if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
         return
     path = history_desc_path(md5sum)
     LOG.debug("Writing upload history: %s", path)
     path.resolve().parent.mkdir(parents=True, exist_ok=True)
-    history: T.Dict[str, T.Any] = {
-        "params": params,
-        "summary": summary,
-    }
+    history: dict[str, T.Any] = {"params": params, "summary": summary}
     if metadatas is not None:
-        history["descs"] = [types.as_desc(metadata) for metadata in metadatas]
+        history["descs"] = [
+            DescriptionJSONSerializer.as_desc(metadata) for metadata in metadatas
+        ]
     with open(path, "w") as fp:
         fp.write(json.dumps(history))
+
+
+class PersistentCache:
+    _lock: contextlib.nullcontext | threading.Lock
+
+    def __init__(self, file: str):
+        # SQLite3 backend supports concurrent access without a lock
+        if dbm.whichdb(file) == "dbm.sqlite3":
+            self._lock = contextlib.nullcontext()
+        else:
+            self._lock = threading.Lock()
+        self._file = file
+
+    def get(self, key: str) -> str | None:
+        s = time.perf_counter()
+
+        with self._lock:
+            with dbm.open(self._file, flag="c") as db:
+                value: bytes | None = db.get(key)
+
+        if value is None:
+            return None
+
+        payload = self._decode(value)
+
+        if self._is_expired(payload):
+            return None
+
+        file_handle = payload.get("file_handle")
+
+        LOG.debug(
+            f"Found file handle for {key} in cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
+        )
+
+        return T.cast(str, file_handle)
+
+    def set(self, key: str, file_handle: str, expires_in: int = 3600 * 24 * 2) -> None:
+        s = time.perf_counter()
+
+        payload = {
+            "expires_at": time.time() + expires_in,
+            "file_handle": file_handle,
+        }
+
+        value: bytes = json.dumps(payload).encode("utf-8")
+
+        with self._lock:
+            with dbm.open(self._file, flag="c") as db:
+                db[key] = value
+
+        LOG.debug(
+            f"Cached file handle for {key} ({(time.perf_counter() - s) * 1000:.0f} ms)"
+        )
+
+    def clear_expired(self) -> list[str]:
+        s = time.perf_counter()
+
+        expired_keys: list[str] = []
+
+        with self._lock:
+            with dbm.open(self._file, flag="c") as db:
+                if hasattr(db, "items"):
+                    items: T.Iterable[tuple[str | bytes, bytes]] = db.items()
+                else:
+                    items = ((key, db[key]) for key in db.keys())
+
+                for key, value in items:
+                    payload = self._decode(value)
+                    if self._is_expired(payload):
+                        del db[key]
+                        expired_keys.append(T.cast(str, key))
+
+        if expired_keys:
+            LOG.debug(
+                f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
+            )
+
+        return expired_keys
+
+    def _is_expired(self, payload: JSONDict) -> bool:
+        expires_at = payload.get("expires_at")
+        if isinstance(expires_at, (int, float)):
+            return expires_at is None or expires_at <= time.time()
+        return False
+
+    def _decode(self, value: bytes) -> JSONDict:
+        try:
+            payload = json.loads(value.decode("utf-8"))
+        except json.JSONDecodeError as ex:
+            LOG.warning(f"Failed to decode cache value: {ex}")
+            return {}
+
+        if not isinstance(payload, dict):
+            LOG.warning(f"Invalid cache value format: {payload}")
+            return {}
+
+        return payload

mapillary_tools/mp4/construct_mp4_parser.py

@@ -1,4 +1,5 @@
 # pyre-ignore-all-errors[5, 16, 21, 58]
+from __future__ import annotations
 
 import typing as T
 
@@ -42,7 +43,7 @@ BoxType = T.Literal[
 
 class BoxDict(T.TypedDict, total=True):
     type: BoxType
-    data: T.Union[T.Sequence["BoxDict"], T.Dict[str, T.Any], bytes]
+    data: T.Sequence["BoxDict"] | dict[str, T.Any] | bytes
 
 
 _UNITY_MATRIX = [0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000]
@@ -369,6 +370,10 @@ BoxHeader64 = C.Struct(
 SwitchMapType = T.Dict[BoxType, T.Union[C.Construct, "SwitchMapType"]]
 
 
+class BoxNotFoundError(Exception):
+    pass
+
+
 class Box64ConstructBuilder:
     """
     Build a box struct that **parses** MP4 boxes with both 32-bit and 64-bit sizes.
@@ -376,7 +381,7 @@ class Box64ConstructBuilder:
     NOTE: Do not build data with this struct. For building, use Box32StructBuilder instead.
     """
 
-    _box: T.Optional[C.Construct]
+    _box: C.Construct | None
 
     def __init__(
         self,
@@ -438,7 +443,7 @@ class Box64ConstructBuilder:
     def parse_box(self, data: bytes) -> BoxDict:
         return T.cast(BoxDict, self.Box.parse(data))
 
-    def parse_boxlist(self, data: bytes) -> T.List[BoxDict]:
+    def parse_boxlist(self, data: bytes) -> list[BoxDict]:
         return T.cast(T.List[BoxDict], self.BoxList.parse(data))
 
 
@@ -464,7 +469,7 @@ class Box32ConstructBuilder(Box64ConstructBuilder):
     def parse_box(self, data: bytes) -> BoxDict:
         raise NotImplementedError("Box32ConstructBuilder does not support parsing")
 
-    def parse_boxlist(self, data: bytes) -> T.List[BoxDict]:
+    def parse_boxlist(self, data: bytes) -> list[BoxDict]:
         raise NotImplementedError("Box32ConstructBuilder does not support parsing")
 
     def build_box(self, box: BoxDict) -> bytes:
@@ -566,7 +571,9 @@ def _new_cmap_without_boxes(
 # pyre-ignore[9]: pyre does not support recursive type SwitchMapType
 MP4_WITHOUT_STBL_CMAP: SwitchMapType = {
     # pyre-ignore[6]: pyre does not support recursive type SwitchMapType
-    b"moov": _new_cmap_without_boxes(CMAP[b"moov"], [b"stbl"]),
+    b"moov": _new_cmap_without_boxes(
+        CMAP[b"moov"], T.cast(T.Sequence[BoxType], [b"stbl"])
+    ),
 }
 
 # for parsing mp4 only
@@ -584,17 +591,17 @@ MOOVWithoutSTBLBuilderConstruct = Box32ConstructBuilder(
 
 
 def find_box_at_pathx(
-    box: T.Union[T.Sequence[BoxDict], BoxDict], path: T.Sequence[bytes]
+    box: T.Sequence[BoxDict] | BoxDict, path: T.Sequence[bytes]
 ) -> BoxDict:
     found = find_box_at_path(box, path)
     if found is None:
-        raise ValueError(f"box at path {path} not found")
+        raise BoxNotFoundError(f"box at path {path} not found")
     return found
 
 
 def find_box_at_path(
-    box: T.Union[T.Sequence[BoxDict], BoxDict], path: T.Sequence[bytes]
-) -> T.Optional[BoxDict]:
+    box: T.Sequence[BoxDict] | BoxDict, path: T.Sequence[bytes]
+) -> BoxDict | None:
     if not path:
         return None
 
@@ -608,7 +615,7 @@ def find_box_at_path(
         if box["type"] == path[0]:
             if len(path) == 1:
                 return box
-            box_data = T.cast(T.Sequence[BoxDict], box["data"])
+            box_data = T.cast(T.List[BoxDict], box["data"])
             # ListContainer from construct is not sequence
             assert isinstance(box_data, T.Sequence), (
                 f"expect a list of boxes but got {type(box_data)} at path {path}"

mapillary_tools/mp4/io_utils.py

@@ -3,7 +3,6 @@ import typing as T
 
 
 class ChainedIO(io.IOBase):
-    # is the chained stream seekable?
     _streams: T.Sequence[io.IOBase]
     # the beginning offset of the current stream
     _begin_offset: int

mapillary_tools/mp4/mp4_sample_parser.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import typing as T
 from pathlib import Path
@@ -42,16 +44,16 @@ class Sample(T.NamedTuple):
     exact_timedelta: float
 
     # reference to the sample description
-    description: T.Dict
+    description: dict
 
 
 def _extract_raw_samples(
     sizes: T.Sequence[int],
-    chunk_entries: T.Sequence[T.Dict],
+    chunk_entries: T.Sequence[dict],
     chunk_offsets: T.Sequence[int],
     timedeltas: T.Sequence[int],
-    composition_offsets: T.Optional[T.Sequence[int]],
-    syncs: T.Optional[T.Set[int]],
+    composition_offsets: list[int] | None,
+    syncs: set[int] | None,
 ) -> T.Generator[RawSample, None, None]:
     if not sizes:
         return
@@ -128,7 +130,7 @@ def _extract_raw_samples(
 
 def _extract_samples(
     raw_samples: T.Iterator[RawSample],
-    descriptions: T.List,
+    descriptions: list,
     timescale: int,
 ) -> T.Generator[Sample, None, None]:
     acc_delta = 0
@@ -152,21 +154,21 @@ STBLBoxlistConstruct = cparser.Box64ConstructBuilder(
 
 def extract_raw_samples_from_stbl_data(
     stbl: bytes,
-) -> T.Tuple[T.List[T.Dict], T.Generator[RawSample, None, None]]:
-    descriptions = []
-    sizes = []
-    chunk_offsets = []
-    chunk_entries = []
-    timedeltas: T.List[int] = []
-    composition_offsets: T.Optional[T.List[int]] = None
-    syncs: T.Optional[T.Set[int]] = None
+) -> tuple[list[dict], T.Generator[RawSample, None, None]]:
+    descriptions: list[dict] = []
+    sizes: list[int] = []
+    chunk_offsets: list[int] = []
+    chunk_entries: list[dict] = []
+    timedeltas: list[int] = []
+    composition_offsets: list[int] | None = None
+    syncs: set[int] | None = None
 
     stbl_children = T.cast(
         T.Sequence[cparser.BoxDict], STBLBoxlistConstruct.parse(stbl)
     )
 
     for box in stbl_children:
-        data: T.Dict = T.cast(T.Dict, box["data"])
+        data: dict = T.cast(dict, box["data"])
 
         if box["type"] == b"stsd":
             descriptions = list(data["entries"])
@@ -225,32 +227,32 @@ class TrackBoxParser:
         )
         self.stbl_data = T.cast(bytes, stbl["data"])
 
-    def extract_tkhd_boxdata(self) -> T.Dict:
+    def extract_tkhd_boxdata(self) -> dict:
         return T.cast(
-            T.Dict, cparser.find_box_at_pathx(self.trak_children, [b"tkhd"])["data"]
+            dict, cparser.find_box_at_pathx(self.trak_children, [b"tkhd"])["data"]
         )
 
     def is_video_track(self) -> bool:
         hdlr = cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"hdlr"])
         return T.cast(T.Dict[str, T.Any], hdlr["data"])["handler_type"] == b"vide"
 
-    def extract_sample_descriptions(self) -> T.List[T.Dict]:
+    def extract_sample_descriptions(self) -> list[dict]:
         # TODO: return [] if parsing fail
         boxes = _STSDBoxListConstruct.parse(self.stbl_data)
         stsd = cparser.find_box_at_pathx(
             T.cast(T.Sequence[cparser.BoxDict], boxes), [b"stsd"]
         )
-        return T.cast(T.List[T.Dict], T.cast(T.Dict, stsd["data"])["entries"])
+        return T.cast(T.List[dict], T.cast(dict, stsd["data"])["entries"])
 
-    def extract_elst_boxdata(self) -> T.Optional[T.Dict]:
+    def extract_elst_boxdata(self) -> dict | None:
         box = cparser.find_box_at_path(self.trak_children, [b"edts", b"elst"])
         if box is None:
             return None
-        return T.cast(T.Dict, box["data"])
+        return T.cast(dict, box["data"])
 
-    def extract_mdhd_boxdata(self) -> T.Dict:
+    def extract_mdhd_boxdata(self) -> dict:
         box = cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"mdhd"])
-        return T.cast(T.Dict, box["data"])
+        return T.cast(dict, box["data"])
 
     def extract_raw_samples(self) -> T.Generator[RawSample, None, None]:
         _, raw_samples = extract_raw_samples_from_stbl_data(self.stbl_data)
@@ -259,7 +261,7 @@ class TrackBoxParser:
     def extract_samples(self) -> T.Generator[Sample, None, None]:
         descriptions, raw_samples = extract_raw_samples_from_stbl_data(self.stbl_data)
         mdhd = T.cast(
-            T.Dict,
+            dict,
             cparser.find_box_at_pathx(self.trak_children, [b"mdia", b"mdhd"])["data"],
         )
         yield from _extract_samples(raw_samples, descriptions, mdhd["timescale"])
@@ -278,16 +280,22 @@ class MovieBoxParser:
     def parse_file(cls, video_path: Path) -> "MovieBoxParser":
         with video_path.open("rb") as fp:
             moov = sparser.parse_box_data_firstx(fp, [b"moov"])
-        return MovieBoxParser(moov)
+        return cls(moov)
 
     @classmethod
     def parse_stream(cls, stream: T.BinaryIO) -> "MovieBoxParser":
         moov = sparser.parse_box_data_firstx(stream, [b"moov"])
-        return MovieBoxParser(moov)
+        return cls(moov)
 
-    def extract_mvhd_boxdata(self) -> T.Dict:
+    def extract_mvhd_boxdata(self) -> dict:
         mvhd = cparser.find_box_at_pathx(self.moov_children, [b"mvhd"])
-        return T.cast(T.Dict, mvhd["data"])
+        return T.cast(dict, mvhd["data"])
+
+    def extract_udta_boxdata(self) -> dict | None:
+        box = cparser.find_box_at_path(self.moov_children, [b"udta"])
+        if box is None:
+            return None
+        return T.cast(dict, box["data"])
 
     def extract_tracks(self) -> T.Generator[TrackBoxParser, None, None]:
         for box in self.moov_children:
@@ -312,7 +320,7 @@ class MovieBoxParser:
         return TrackBoxParser(trak_children)
 
 
-_DT_1904 = datetime.datetime.utcfromtimestamp(0).replace(year=1904)
+_DT_1904 = datetime.datetime.fromtimestamp(0, datetime.timezone.utc).replace(year=1904)
 
 
 def to_datetime(seconds_since_1904: int) -> datetime.datetime:

mapillary_tools/mp4/simple_mp4_builder.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import io
 import typing as T
@@ -64,8 +66,8 @@ class _SampleChunk:
     offset: int
 
 
-def _build_chunks(raw_samples: T.Iterable[RawSample]) -> T.List[_SampleChunk]:
-    chunks: T.List[_SampleChunk] = []
+def _build_chunks(raw_samples: T.Iterable[RawSample]) -> list[_SampleChunk]:
+    chunks: list[_SampleChunk] = []
     prev_raw_sample = None
 
     for raw_sample in raw_samples:
@@ -120,7 +122,7 @@ class _CompressedSampleDelta:
 
 def _build_stts(sample_deltas: T.Iterable[int]) -> BoxDict:
     # compress deltas
-    compressed: T.List[_CompressedSampleDelta] = []
+    compressed: list[_CompressedSampleDelta] = []
     for delta in sample_deltas:
         if compressed and delta == compressed[-1].sample_delta:
             compressed[-1].sample_count += 1
@@ -146,7 +148,7 @@ class _CompressedSampleCompositionOffset:
 
 def _build_ctts(sample_composition_offsets: T.Iterable[int]) -> BoxDict:
     # compress offsets
-    compressed: T.List[_CompressedSampleCompositionOffset] = []
+    compressed: list[_CompressedSampleCompositionOffset] = []
     for offset in sample_composition_offsets:
         if compressed and offset == compressed[-1].sample_offset:
             compressed[-1].sample_count += 1
@@ -182,7 +184,7 @@ def _build_stss(is_syncs: T.Iterable[bool]) -> BoxDict:
 
 def build_stbl_from_raw_samples(
     descriptions: T.Sequence[T.Any], raw_samples: T.Iterable[RawSample]
-) -> T.List[BoxDict]:
+) -> list[BoxDict]:
     # raw_samples could be iterator so convert to list
     raw_samples = list(raw_samples)
     # It is recommended that the boxes within the Sample Table Box be in the following order:
@@ -329,9 +331,8 @@ _MOOVChildrenParserConstruct = cparser.Box64ConstructBuilder(
 
 def transform_mp4(
     src_fp: T.BinaryIO,
-    sample_generator: T.Optional[
-        T.Callable[[T.BinaryIO, T.List[BoxDict]], T.Iterator[io.IOBase]]
-    ] = None,
+    sample_generator: T.Callable[[T.BinaryIO, list[BoxDict]], T.Iterator[io.IOBase]]
+    | None = None,
 ) -> io_utils.ChainedIO:
     # extract ftyp
     src_fp.seek(0)
@@ -347,7 +348,7 @@ def transform_mp4(
 
     # extract video samples
     source_samples = list(iterate_samples(moov_children))
-    sample_readers: T.List[io.IOBase] = [
+    sample_readers: list[io.IOBase] = [
         io_utils.SlicedIO(src_fp, sample.offset, sample.size)
         for sample in source_samples
     ]

mapillary_tools/mp4/simple_mp4_parser.py

@@ -1,4 +1,5 @@
 # pyre-ignore-all-errors[5, 16, 21, 24, 58]
+from __future__ import annotations
 
 import io
 import typing as T
@@ -130,8 +131,8 @@ def parse_boxes_recursive(
     stream: T.BinaryIO,
     maxsize: int = -1,
     depth: int = 0,
-    box_list_types: T.Optional[T.Set[bytes]] = None,
-) -> T.Generator[T.Tuple[Header, int, T.BinaryIO], None, None]:
+    box_list_types: set[bytes] | None = None,
+) -> T.Generator[tuple[Header, int, T.BinaryIO], None, None]:
     assert maxsize == -1 or 0 <= maxsize
 
     if box_list_types is None:
@@ -152,10 +153,10 @@ def parse_boxes_recursive(
 
 def parse_path(
     stream: T.BinaryIO,
-    path: T.Sequence[T.Union[bytes, T.Sequence[bytes]]],
+    path: T.Sequence[bytes | T.Sequence[bytes]],
     maxsize: int = -1,
     depth: int = 0,
-) -> T.Generator[T.Tuple[Header, T.BinaryIO], None, None]:
+) -> T.Generator[tuple[Header, T.BinaryIO], None, None]:
     if not path:
         return
 
@@ -172,8 +173,8 @@ def parse_path(
 
 
 def _parse_path_first(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1, depth: int = 0
-) -> T.Optional[T.Tuple[Header, T.BinaryIO]]:
+    stream: T.BinaryIO, path: list[bytes], maxsize: int = -1, depth: int = 0
+) -> tuple[Header, T.BinaryIO] | None:
     if not path:
         return None
     for h, s in parse_boxes(stream, maxsize=maxsize, extend_eof=depth == 0):
@@ -187,19 +188,9 @@ def _parse_path_first(
     return None
 
 
-def parse_box_path_firstx(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1
-) -> T.Tuple[Header, T.BinaryIO]:
-    # depth=1 will disable EoF extension
-    parsed = _parse_path_first(stream, path, maxsize=maxsize, depth=1)
-    if parsed is None:
-        raise BoxNotFoundError(f"unable find box at path {path}")
-    return parsed
-
-
 def parse_mp4_data_first(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1
-) -> T.Optional[bytes]:
+    stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
+) -> bytes | None:
     # depth=0 will enable EoF extension
     parsed = _parse_path_first(stream, path, maxsize=maxsize, depth=0)
     if parsed is None:
@@ -209,7 +200,7 @@ def parse_mp4_data_first(
 
 
 def parse_mp4_data_firstx(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1
+    stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
 ) -> bytes:
     data = parse_mp4_data_first(stream, path, maxsize=maxsize)
     if data is None:
@@ -218,8 +209,8 @@ def parse_mp4_data_firstx(
 
 
 def parse_box_data_first(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1
-) -> T.Optional[bytes]:
+    stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
+) -> bytes | None:
     # depth=1 will disable EoF extension
     parsed = _parse_path_first(stream, path, maxsize=maxsize, depth=1)
     if parsed is None:
@@ -229,7 +220,7 @@ def parse_box_data_first(
 
 
 def parse_box_data_firstx(
-    stream: T.BinaryIO, path: T.List[bytes], maxsize: int = -1
+    stream: T.BinaryIO, path: list[bytes], maxsize: int = -1
 ) -> bytes:
     data = parse_box_data_first(stream, path, maxsize=maxsize)
     if data is None: