mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/uploader.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
+import concurrent.futures
 import dataclasses
 import io
 import json
 import logging
 import os
+import queue
 import struct
+import sys
 import tempfile
+import threading
 import time
 import typing as T
 import uuid
@@ -14,14 +18,57 @@ import zipfile
 from contextlib import contextmanager
 from pathlib import Path
 
+if sys.version_info >= (3, 11):
+    from typing import Required
+else:
+    from typing_extensions import Required
+
 import requests
 
-from . import api_v4, constants, exif_write, types, upload_api_v4
+from . import (
+    api_v4,
+    config,
+    constants,
+    exif_write,
+    geo,
+    history,
+    telemetry,
+    types,
+    upload_api_v4,
+    utils,
+)
+from .camm import camm_builder, camm_parser
+from .gpmf import gpmf_parser
+from .mp4 import simple_mp4_builder
+from .serializer.description import (
+    desc_file_to_exif,
+    DescriptionJSONSerializer,
+    validate_image_desc,
+)
 
 
 LOG = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass(frozen=True)
+class UploadOptions:
+    user_items: config.UserItem
+    chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024)
+    num_upload_workers: int = constants.MAX_IMAGE_UPLOAD_WORKERS
+    dry_run: bool = False
+    nofinish: bool = False
+    noresume: bool = False
+
+    def __post_init__(self):
+        if self.num_upload_workers <= 0:
+            raise ValueError(
+                f"Expect positive num_upload_workers but got {self.num_upload_workers}"
+            )
+
+        if self.chunk_size <= 0:
+            raise ValueError(f"Expect positive chunk_size but got {self.chunk_size}")
+
+
 class UploaderProgress(T.TypedDict, total=True):
     """
     Progress data that Uploader cares about.
@@ -46,7 +93,7 @@ class UploaderProgress(T.TypedDict, total=True):
     #   - offset == entity_size when "upload_end" or "upload_finished"
     entity_size: int
 
-    # An "upload_interrupted" will increase it. Reset to 0 if a chunk is uploaded
+    # An "upload_retrying" will increase it. Reset to 0 if a chunk is uploaded
     retries: int
 
     # Cluster ID after finishing the upload
@@ -56,17 +103,21 @@ class UploaderProgress(T.TypedDict, total=True):
 class SequenceProgress(T.TypedDict, total=False):
     """Progress data at sequence level"""
 
-    # md5sum of the zipfile/BlackVue/CAMM in uploading
-    md5sum: str
+    # Used to check if it is uploaded or not
+    sequence_md5sum: Required[str]
+
+    # Used to resume from the previous upload,
+    # so it has to an unique identifier (hash) of the upload content
+    upload_md5sum: str
 
     # File type
-    file_type: str
+    file_type: Required[str]
 
     # How many sequences in total. It's always 1 when uploading Zipfile/BlackVue/CAMM
-    total_sequence_count: int
+    total_sequence_count: Required[int]
 
     # 0-based nth sequence. It is always 0 when uploading Zipfile/BlackVue/CAMM
-    sequence_idx: int
+    sequence_idx: Required[int]
 
     # How many images in the sequence. It's available only when uploading directories/Zipfiles
     sequence_image_count: int
@@ -74,7 +125,7 @@ class SequenceProgress(T.TypedDict, total=False):
     # MAPSequenceUUID. It is only available for directory uploading
     sequence_uuid: str
 
-    # Path to the Zipfile/BlackVue/CAMM
+    # Path to the image/video/zip
     import_path: str
 
 
@@ -102,13 +153,43 @@ class InvalidMapillaryZipFileError(SequenceError):
     pass
 
 
+# BELOW demonstrates the pseudocode for a typical upload workflow
+# and when upload events are emitted
+#################################################################
+# def pseudo_upload(metadata):
+#     emit("upload_start")
+#     while True:
+#         try:
+#             if is_sequence(metadata):
+#                 for image in metadata:
+#                     upload_stream(image.read())
+#                     emit("upload_progress")
+#             elif is_video(metadata):
+#                 offset = fetch_offset()
+#                 emit("upload_fetch_offset")
+#                 for chunk in metadata.read()[offset:]:
+#                     upload_stream(chunk)
+#                     emit("upload_progress")
+#         except BaseException as ex:  # Include KeyboardInterrupt
+#             if retryable(ex):
+#                 emit("upload_retrying")
+#                 continue
+#             else:
+#                 emit("upload_failed")
+#                 raise ex
+#         else:
+#             break
+#     emit("upload_end")
+#     finish_upload(data)
+#     emit("upload_finished")
 EventName = T.Literal[
     "upload_start",
     "upload_fetch_offset",
     "upload_progress",
+    "upload_retrying",
     "upload_end",
+    "upload_failed",
     "upload_finished",
-    "upload_interrupted",
 ]
 
 
@@ -121,6 +202,7 @@ class EventEmitter:
     def on(self, event: EventName):
         def _wrap(callback):
             self.events.setdefault(event, []).append(callback)
+            return callback
 
         return _wrap
 
@@ -135,7 +217,131 @@ class UploadResult:
     error: Exception | None = None
 
 
-class ZipImageSequence:
+class VideoUploader:
+    @classmethod
+    def upload_videos(
+        cls, mly_uploader: Uploader, video_metadatas: T.Sequence[types.VideoMetadata]
+    ) -> T.Generator[tuple[types.VideoMetadata, UploadResult], None, None]:
+        # If upload in a random order, then interrupted uploads has a higher chance to expire.
+        # Therefore sort videos to make sure interrupted uploads are resumed as early as possible
+        sorted_video_metadatas = sorted(video_metadatas, key=lambda m: m.filename)
+
+        for idx, video_metadata in enumerate(sorted_video_metadatas):
+            LOG.debug(f"Checksum for video {video_metadata.filename}...")
+            try:
+                video_metadata.update_md5sum()
+            except Exception as ex:
+                yield video_metadata, UploadResult(error=ex)
+                continue
+
+            assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
+
+            progress: SequenceProgress = {
+                "total_sequence_count": len(sorted_video_metadatas),
+                "sequence_idx": idx,
+                "file_type": video_metadata.filetype.value,
+                "import_path": str(video_metadata.filename),
+                "sequence_md5sum": video_metadata.md5sum,
+            }
+
+            try:
+                with cls.build_camm_stream(video_metadata) as camm_fp:
+                    # Upload the mp4 stream
+                    file_handle = mly_uploader.upload_stream(
+                        T.cast(T.IO[bytes], camm_fp),
+                        progress=T.cast(T.Dict[str, T.Any], progress),
+                    )
+
+                cluster_id = mly_uploader.finish_upload(
+                    file_handle,
+                    api_v4.ClusterFileType.CAMM,
+                    progress=T.cast(T.Dict[str, T.Any], progress),
+                )
+            except Exception as ex:
+                yield video_metadata, UploadResult(error=ex)
+            else:
+                yield video_metadata, UploadResult(result=cluster_id)
+
+    @classmethod
+    @contextmanager
+    def build_camm_stream(cls, video_metadata: types.VideoMetadata):
+        # Convert video metadata to CAMMInfo
+        camm_info = cls.prepare_camm_info(video_metadata)
+
+        # Create the CAMM sample generator
+        camm_sample_generator = camm_builder.camm_sample_generator2(camm_info)
+
+        with video_metadata.filename.open("rb") as src_fp:
+            # Build the mp4 stream with the CAMM samples
+            yield simple_mp4_builder.transform_mp4(src_fp, camm_sample_generator)
+
+    @classmethod
+    def prepare_camm_info(
+        cls, video_metadata: types.VideoMetadata
+    ) -> camm_parser.CAMMInfo:
+        camm_info = camm_parser.CAMMInfo(
+            make=video_metadata.make or "", model=video_metadata.model or ""
+        )
+
+        for point in video_metadata.points:
+            if isinstance(point, telemetry.CAMMGPSPoint):
+                if camm_info.gps is None:
+                    camm_info.gps = []
+                camm_info.gps.append(point)
+
+            elif isinstance(point, telemetry.GPSPoint):
+                # There is no proper CAMM entry for GoPro GPS
+                if camm_info.mini_gps is None:
+                    camm_info.mini_gps = []
+                camm_info.mini_gps.append(point)
+
+            elif isinstance(point, geo.Point):
+                if camm_info.mini_gps is None:
+                    camm_info.mini_gps = []
+                camm_info.mini_gps.append(point)
+            else:
+                raise ValueError(f"Unknown point type: {point}")
+
+        if constants.MAPILLARY__EXPERIMENTAL_ENABLE_IMU:
+            if video_metadata.filetype is types.FileType.GOPRO:
+                with video_metadata.filename.open("rb") as fp:
+                    gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True)
+                if gopro_info is not None:
+                    camm_info.accl = gopro_info.accl or []
+                    camm_info.gyro = gopro_info.gyro or []
+                    camm_info.magn = gopro_info.magn or []
+
+        return camm_info
+
+
+class ZipUploader:
+    @classmethod
+    def upload_zipfiles(
+        cls, mly_uploader: Uploader, zip_paths: T.Sequence[Path]
+    ) -> T.Generator[tuple[Path, UploadResult], None, None]:
+        # If upload in a random order, then interrupted uploads has a higher chance to expire.
+        # Therefore sort zipfiles to make sure interrupted uploads are resumed as early as possible
+        sorted_zip_paths = sorted(zip_paths)
+
+        for idx, zip_path in enumerate(sorted_zip_paths):
+            progress: SequenceProgress = {
+                "total_sequence_count": len(sorted_zip_paths),
+                "sequence_idx": idx,
+                "import_path": str(zip_path),
+                "file_type": types.FileType.ZIP.value,
+                "sequence_md5sum": "",  # Placeholder, will be set in upload_zipfile
+            }
+            try:
+                cluster_id = cls._upload_zipfile(
+                    mly_uploader,
+                    zip_path,
+                    progress=T.cast(T.Dict[str, T.Any], progress),
+                )
+            except Exception as ex:
+                yield zip_path, UploadResult(error=ex)
+            else:
+                yield zip_path, UploadResult(result=cluster_id)
+
     @classmethod
     def zip_images(
         cls, metadatas: T.Sequence[types.ImageMetadata], zip_dir: Path
@@ -148,30 +354,105 @@ class ZipImageSequence:
 
         for sequence_uuid, sequence in sequences.items():
             _validate_metadatas(sequence)
-            upload_md5sum = types.update_sequence_md5sum(sequence)
-
             # For atomicity we write into a WIP file and then rename to the final file
             wip_zip_filename = zip_dir.joinpath(
                 f".mly_zip_{uuid.uuid4()}_{sequence_uuid}_{os.getpid()}_{int(time.time())}"
             )
-            filename = _session_key(upload_md5sum, upload_api_v4.ClusterFileType.ZIP)
-            zip_filename = zip_dir.joinpath(filename)
-            with wip_file_context(wip_zip_filename, zip_filename) as wip_path:
+            with cls._wip_file_context(wip_zip_filename) as wip_path:
                 with wip_path.open("wb") as wip_fp:
-                    actual_md5sum = cls.zip_sequence_deterministically(sequence, wip_fp)
-                    assert actual_md5sum == upload_md5sum, "md5sum mismatch"
+                    cls._zip_sequence_fp(sequence, wip_fp)
+
+    @classmethod
+    def zip_images_and_upload(
+        cls, uploader: Uploader, image_metadatas: T.Sequence[types.ImageMetadata]
+    ) -> T.Generator[tuple[str, UploadResult], None, None]:
+        sequences = types.group_and_sort_images(image_metadatas)
+
+        for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
+            try:
+                _validate_metadatas(sequence)
+            except Exception as ex:
+                yield sequence_uuid, UploadResult(error=ex)
+                continue
+
+            with tempfile.NamedTemporaryFile() as fp:
+                try:
+                    sequence_md5sum = cls._zip_sequence_fp(sequence, fp)
+                except Exception as ex:
+                    yield sequence_uuid, UploadResult(error=ex)
+                    continue
+
+                sequence_progress: SequenceProgress = {
+                    "sequence_idx": sequence_idx,
+                    "total_sequence_count": len(sequences),
+                    "sequence_image_count": len(sequence),
+                    "sequence_uuid": sequence_uuid,
+                    "file_type": types.FileType.ZIP.value,
+                    "sequence_md5sum": sequence_md5sum,
+                }
+
+                try:
+                    file_handle = uploader.upload_stream(
+                        fp, progress=T.cast(T.Dict[str, T.Any], sequence_progress)
+                    )
+                    cluster_id = uploader.finish_upload(
+                        file_handle,
+                        api_v4.ClusterFileType.ZIP,
+                        progress=T.cast(T.Dict[str, T.Any], sequence_progress),
+                    )
+                except Exception as ex:
+                    yield sequence_uuid, UploadResult(error=ex)
+                    continue
+
+            yield sequence_uuid, UploadResult(result=cluster_id)
+
+    @classmethod
+    def _upload_zipfile(
+        cls,
+        uploader: Uploader,
+        zip_path: Path,
+        progress: dict[str, T.Any] | None = None,
+    ) -> str:
+        if progress is None:
+            progress = {}
+
+        with zipfile.ZipFile(zip_path) as ziph:
+            namelist = ziph.namelist()
+            if not namelist:
+                raise InvalidMapillaryZipFileError("Zipfile has no files")
+
+        with zip_path.open("rb") as zip_fp:
+            sequence_md5sum = cls._extract_sequence_md5sum(zip_fp)
+
+        # Send the copy of the input progress to each upload session, to avoid modifying the original one
+        mutable_progress: SequenceProgress = {
+            **T.cast(SequenceProgress, progress),
+            "sequence_image_count": len(namelist),
+            "sequence_md5sum": sequence_md5sum,
+            "file_type": types.FileType.ZIP.value,
+        }
+
+        with zip_path.open("rb") as zip_fp:
+            file_handle = uploader.upload_stream(
+                zip_fp, progress=T.cast(T.Dict[str, T.Any], mutable_progress)
+            )
+
+        cluster_id = uploader.finish_upload(
+            file_handle,
+            api_v4.ClusterFileType.ZIP,
+            progress=T.cast(T.Dict[str, T.Any], mutable_progress),
+        )
+
+        return cluster_id
 
     @classmethod
-    def zip_sequence_deterministically(
+    def _zip_sequence_fp(
         cls,
         sequence: T.Sequence[types.ImageMetadata],
         zip_fp: T.IO[bytes],
     ) -> str:
         """
-        Write a sequence of ImageMetadata into the zipfile handle. It should guarantee
-        that the same sequence always produces the same zipfile, because the
-        sequence md5sum will be used to upload the zipfile or resume the upload.
-
+        Write a sequence of ImageMetadata into the zipfile handle.
         The sequence has to be one sequence and sorted.
         """
 
@@ -180,21 +461,27 @@ class ZipImageSequence:
             f"Only one sequence is allowed but got {len(sequence_groups)}: {list(sequence_groups.keys())}"
         )
 
-        upload_md5sum = types.update_sequence_md5sum(sequence)
+        if sequence:
+            LOG.debug(f"Checksum for sequence {sequence[0].MAPSequenceUUID}...")
+        sequence_md5sum = types.update_sequence_md5sum(sequence)
 
         with zipfile.ZipFile(zip_fp, "w", zipfile.ZIP_DEFLATED) as zipf:
             for idx, metadata in enumerate(sequence):
-                # Use {idx}.jpg (suffix does not matter) as the archive name to ensure the
-                # resulting zipfile is deterministic. This determinism is based on the upload_md5sum,
-                # which is derived from a list of image md5sums
-                cls._write_imagebytes_in_zip(zipf, metadata, arcname=f"{idx}.jpg")
+                # Arcname should be unique, the name does not matter
+                arcname = f"{idx}.jpg"
+                zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
+                zipf.writestr(zipinfo, SingleImageUploader.dump_image_bytes(metadata))
             assert len(sequence) == len(set(zipf.namelist()))
-            zipf.comment = json.dumps({"upload_md5sum": upload_md5sum}).encode("utf-8")
+            zipf.comment = json.dumps(
+                {"sequence_md5sum": sequence_md5sum},
+                sort_keys=True,
+                separators=(",", ":"),
+            ).encode("utf-8")
 
-        return upload_md5sum
+        return sequence_md5sum
 
     @classmethod
-    def extract_upload_md5sum(cls, zip_fp: T.IO[bytes]) -> str:
+    def _extract_sequence_md5sum(cls, zip_fp: T.IO[bytes]) -> str:
         with zipfile.ZipFile(zip_fp, "r", zipfile.ZIP_DEFLATED) as ziph:
             comment = ziph.comment
 
@@ -209,162 +496,384 @@ class ZipImageSequence:
         except json.JSONDecodeError as ex:
             raise InvalidMapillaryZipFileError(str(ex)) from ex
 
-        upload_md5sum = zip_metadata.get("upload_md5sum")
+        sequence_md5sum = zip_metadata.get("sequence_md5sum")
 
-        if not upload_md5sum and not isinstance(upload_md5sum, str):
-            raise InvalidMapillaryZipFileError("No upload_md5sum found")
+        if not sequence_md5sum and not isinstance(sequence_md5sum, str):
+            raise InvalidMapillaryZipFileError("No sequence_md5sum found")
 
-        return upload_md5sum
+        return sequence_md5sum
 
     @classmethod
-    def _write_imagebytes_in_zip(
-        cls, zipf: zipfile.ZipFile, metadata: types.ImageMetadata, arcname: str
-    ):
+    @contextmanager
+    def _wip_file_context(cls, wip_path: Path):
         try:
-            edit = exif_write.ExifEdit(metadata.filename)
-        except struct.error as ex:
-            raise ExifError(f"Failed to load EXIF: {ex}", metadata.filename) from ex
-
-        # The cast is to fix the type checker error
-        edit.add_image_description(
-            T.cast(T.Dict, types.desc_file_to_exif(types.as_desc(metadata)))
-        )
-
+            os.remove(wip_path)
+        except FileNotFoundError:
+            pass
         try:
-            image_bytes = edit.dump_image_bytes()
-        except struct.error as ex:
-            raise ExifError(
-                f"Failed to dump EXIF bytes: {ex}", metadata.filename
-            ) from ex
-
-        zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
-        zipf.writestr(zipinfo, image_bytes)
-
-    @classmethod
-    def prepare_zipfile_and_upload(
-        cls,
-        zip_path: Path,
-        uploader: Uploader,
-        progress: dict[str, T.Any] | None = None,
-    ) -> str:
-        if progress is None:
-            progress = {}
+            yield wip_path
 
-        with zipfile.ZipFile(zip_path) as ziph:
-            namelist = ziph.namelist()
-            if not namelist:
-                raise InvalidMapillaryZipFileError("Zipfile has no files")
+            with wip_path.open("rb") as fp:
+                upload_md5sum = utils.md5sum_fp(fp).hexdigest()
 
-        with zip_path.open("rb") as zip_fp:
-            upload_md5sum = cls.extract_upload_md5sum(zip_fp)
+            done_path = wip_path.parent.joinpath(
+                _suffix_session_key(upload_md5sum, api_v4.ClusterFileType.ZIP)
+            )
 
-        sequence_progress: SequenceProgress = {
-            "sequence_image_count": len(namelist),
-            "file_type": types.FileType.ZIP.value,
-            "md5sum": upload_md5sum,
-        }
+            try:
+                os.remove(done_path)
+            except FileNotFoundError:
+                pass
+            wip_path.rename(done_path)
+        finally:
+            try:
+                os.remove(wip_path)
+            except FileNotFoundError:
+                pass
 
-        session_key = _session_key(upload_md5sum, upload_api_v4.ClusterFileType.ZIP)
 
-        with zip_path.open("rb") as zip_fp:
-            return uploader.upload_stream(
-                zip_fp,
-                upload_api_v4.ClusterFileType.ZIP,
-                session_key,
-                # Send the copy of the input progress to each upload session, to avoid modifying the original one
-                progress=T.cast(T.Dict[str, T.Any], {**progress, **sequence_progress}),
-            )
+class ImageSequenceUploader:
+    def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
+        self.upload_options = upload_options
+        self.emitter = emitter
 
-    @classmethod
-    def prepare_images_and_upload(
-        cls,
-        image_metadatas: T.Sequence[types.ImageMetadata],
-        uploader: Uploader,
-        progress: dict[str, T.Any] | None = None,
+    def upload_images(
+        self, image_metadatas: T.Sequence[types.ImageMetadata]
     ) -> T.Generator[tuple[str, UploadResult], None, None]:
-        if progress is None:
-            progress = {}
-
         sequences = types.group_and_sort_images(image_metadatas)
 
         for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
+            LOG.debug(f"Checksum for image sequence {sequence_uuid}...")
+            sequence_md5sum = types.update_sequence_md5sum(sequence)
+
             sequence_progress: SequenceProgress = {
                 "sequence_idx": sequence_idx,
                 "total_sequence_count": len(sequences),
                 "sequence_image_count": len(sequence),
                 "sequence_uuid": sequence_uuid,
                 "file_type": types.FileType.IMAGE.value,
+                "sequence_md5sum": sequence_md5sum,
             }
 
             try:
-                _validate_metadatas(sequence)
+                cluster_id = self._upload_sequence_and_finish(
+                    sequence,
+                    sequence_progress=T.cast(dict[str, T.Any], sequence_progress),
+                )
             except Exception as ex:
                 yield sequence_uuid, UploadResult(error=ex)
-                continue
+            else:
+                yield sequence_uuid, UploadResult(result=cluster_id)
 
-            with tempfile.NamedTemporaryFile() as fp:
-                try:
-                    upload_md5sum = cls.zip_sequence_deterministically(sequence, fp)
-                except Exception as ex:
-                    yield sequence_uuid, UploadResult(error=ex)
-                    continue
+    def _upload_sequence_and_finish(
+        self,
+        sequence: T.Sequence[types.ImageMetadata],
+        sequence_progress: dict[str, T.Any],
+    ) -> str:
+        _validate_metadatas(sequence)
+
+        sequence_progress["entity_size"] = sum(m.filesize or 0 for m in sequence)
+        self.emitter.emit("upload_start", sequence_progress)
+
+        try:
+            # Retries will be handled in the call (but no upload event emissions)
+            image_file_handles = self._upload_images_parallel(
+                sequence, sequence_progress
+            )
+        except BaseException as ex:  # Include KeyboardInterrupt
+            self.emitter.emit("upload_failed", sequence_progress)
+            raise ex
+
+        manifest_file_handle = self._upload_manifest(image_file_handles)
+
+        self.emitter.emit("upload_end", sequence_progress)
+
+        uploader = Uploader(self.upload_options, emitter=self.emitter)
+        cluster_id = uploader.finish_upload(
+            manifest_file_handle,
+            api_v4.ClusterFileType.MLY_BUNDLE_MANIFEST,
+            progress=sequence_progress,
+        )
+
+        return cluster_id
+
+    def _upload_manifest(self, image_file_handles: T.Sequence[str]) -> str:
+        uploader = Uploader(self.upload_options)
 
-                sequence_progress["md5sum"] = upload_md5sum
+        manifest = {
+            "version": "1",
+            "upload_type": "images",
+            "image_handles": image_file_handles,
+        }
+
+        with io.BytesIO() as manifest_fp:
+            manifest_fp.write(
+                json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode(
+                    "utf-8"
+                )
+            )
+            manifest_fp.seek(0, io.SEEK_SET)
+            return uploader.upload_stream(
+                manifest_fp, session_key=f"{_prefixed_uuid4()}.json"
+            )
 
-                session_key = _session_key(
-                    upload_md5sum, upload_api_v4.ClusterFileType.ZIP
+    def _upload_images_parallel(
+        self,
+        sequence: T.Sequence[types.ImageMetadata],
+        sequence_progress: dict[str, T.Any],
+    ) -> list[str]:
+        if not sequence:
+            return []
+
+        max_workers = min(self.upload_options.num_upload_workers, len(sequence))
+
+        # Lock is used to synchronize event emission
+        lock = threading.Lock()
+
+        # Push all images into the queue
+        image_queue: queue.Queue[tuple[int, types.ImageMetadata]] = queue.Queue()
+        for idx, image_metadata in enumerate(sequence):
+            image_queue.put((idx, image_metadata))
+
+        upload_interrupted = threading.Event()
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(
+                    self._upload_images_from_queue,
+                    image_queue,
+                    lock,
+                    upload_interrupted,
+                    sequence_progress,
                 )
+                for _ in range(max_workers)
+            ]
+
+            indexed_image_file_handles = []
+
+            try:
+                for future in futures:
+                    indexed_image_file_handles.extend(future.result())
+            except KeyboardInterrupt as ex:
+                upload_interrupted.set()
+                raise ex
+
+        # All tasks should be done here, so below is more like assertion
+        image_queue.join()
+        if sys.version_info >= (3, 13):
+            image_queue.shutdown()
+
+        file_handles: list[str] = []
+
+        indexed_image_file_handles.sort()
+
+        # Important to guarantee the order
+        assert len(indexed_image_file_handles) == len(sequence)
+        for expected_idx, (idx, file_handle) in enumerate(indexed_image_file_handles):
+            assert expected_idx == idx
+            file_handles.append(file_handle)
 
+        return file_handles
+
+    def _upload_images_from_queue(
+        self,
+        image_queue: queue.Queue[tuple[int, types.ImageMetadata]],
+        lock: threading.Lock,
+        upload_interrupted: threading.Event,
+        sequence_progress: dict[str, T.Any],
+    ) -> list[tuple[int, str]]:
+        indexed_file_handles = []
+
+        with api_v4.create_user_session(
+            self.upload_options.user_items["user_upload_token"]
+        ) as user_session:
+            single_image_uploader = SingleImageUploader(
+                self.upload_options, user_session=user_session
+            )
+
+            while True:
+                # Assert that all images are already pushed into the queue
                 try:
-                    cluster_id = uploader.upload_stream(
-                        fp,
-                        upload_api_v4.ClusterFileType.ZIP,
-                        session_key,
-                        progress=T.cast(
-                            T.Dict[str, T.Any], {**progress, **sequence_progress}
-                        ),
-                    )
-                except Exception as ex:
-                    yield sequence_uuid, UploadResult(error=ex)
-                    continue
+                    idx, image_metadata = image_queue.get_nowait()
+                except queue.Empty:
+                    break
+
+                # Main thread will handle the interruption
+                if upload_interrupted.is_set():
+                    break
+
+                # Create a new mutatble progress to keep the sequence_progress immutable
+                image_progress = {
+                    **sequence_progress,
+                    "import_path": str(image_metadata.filename),
+                }
+
+                # image_progress will be updated during uploading
+                file_handle = single_image_uploader.upload(
+                    image_metadata, image_progress
+                )
 
-            yield sequence_uuid, UploadResult(result=cluster_id)
+                # Update chunk_size (it was constant if set)
+                image_progress["chunk_size"] = image_metadata.filesize
+
+                # Main thread will handle the interruption
+                if upload_interrupted.is_set():
+                    break
+
+                with lock:
+                    self.emitter.emit("upload_progress", image_progress)
+
+                indexed_file_handles.append((idx, file_handle))
+
+                image_queue.task_done()
+
+        return indexed_file_handles
+
+
+class SingleImageUploader:
+    def __init__(
+        self,
+        upload_options: UploadOptions,
+        user_session: requests.Session | None = None,
+    ):
+        self.upload_options = upload_options
+        self.user_session = user_session
+        self.cache = self._maybe_create_persistent_cache_instance(
+            self.upload_options.user_items, upload_options
+        )
+
+    def upload(
+        self, image_metadata: types.ImageMetadata, image_progress: dict[str, T.Any]
+    ) -> str:
+        image_bytes = self.dump_image_bytes(image_metadata)
+
+        uploader = Uploader(self.upload_options, user_session=self.user_session)
+
+        session_key = uploader._gen_session_key(io.BytesIO(image_bytes), image_progress)
+
+        file_handle = self._get_cached_file_handle(session_key)
+
+        if file_handle is None:
+            # image_progress will be updated during uploading
+            file_handle = uploader.upload_stream(
+                io.BytesIO(image_bytes),
+                session_key=session_key,
+                progress=image_progress,
+            )
+            self._set_file_handle_cache(session_key, file_handle)
+
+        return file_handle
+
+    @classmethod
+    def dump_image_bytes(cls, metadata: types.ImageMetadata) -> bytes:
+        try:
+            edit = exif_write.ExifEdit(metadata.filename)
+        except struct.error as ex:
+            raise ExifError(f"Failed to load EXIF: {ex}", metadata.filename) from ex
+
+        # The cast is to fix the type checker error
+        edit.add_image_description(
+            T.cast(
+                T.Dict, desc_file_to_exif(DescriptionJSONSerializer.as_desc(metadata))
+            )
+        )
+
+        try:
+            return edit.dump_image_bytes()
+        except struct.error as ex:
+            raise ExifError(
+                f"Failed to dump EXIF bytes: {ex}", metadata.filename
+            ) from ex
+
+    @classmethod
+    def _maybe_create_persistent_cache_instance(
+        cls, user_items: config.UserItem, upload_options: UploadOptions
+    ) -> history.PersistentCache | None:
+        if not constants.UPLOAD_CACHE_DIR:
+            LOG.debug(
+                "Upload cache directory is set empty, skipping caching upload file handles"
+            )
+            return None
+
+        if upload_options.dry_run:
+            LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
+            return None
+
+        cache_path_dir = (
+            Path(constants.UPLOAD_CACHE_DIR)
+            .joinpath(api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"))
+            .joinpath(
+                user_items.get("MAPSettingsUserKey", user_items["user_upload_token"])
+            )
+        )
+        cache_path_dir.mkdir(parents=True, exist_ok=True)
+        cache_path = cache_path_dir.joinpath("cached_file_handles")
+
+        # Sanitize sensitive segments for logging
+        sanitized_cache_path = (
+            Path(constants.UPLOAD_CACHE_DIR)
+            .joinpath("***")
+            .joinpath("***")
+            .joinpath("cached_file_handles")
+        )
+        LOG.debug(f"File handle cache path: {sanitized_cache_path}")
+
+        cache = history.PersistentCache(str(cache_path.resolve()))
+        cache.clear_expired()
+
+        return cache
+
+    def _get_cached_file_handle(self, key: str) -> str | None:
+        if self.cache is None:
+            return None
+
+        if _is_uuid(key):
+            return None
+
+        return self.cache.get(key)
+
+    def _set_file_handle_cache(self, key: str, value: str) -> None:
+        if self.cache is None:
+            return
+
+        if _is_uuid(key):
+            return
+
+        self.cache.set(key, value)
 
 
 class Uploader:
     def __init__(
         self,
-        user_items: types.UserItem,
+        upload_options: UploadOptions,
+        user_session: requests.Session | None = None,
         emitter: EventEmitter | None = None,
-        chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
-        dry_run=False,
     ):
-        self.user_items = user_items
+        self.upload_options = upload_options
+        self.user_session = user_session
         if emitter is None:
             # An empty event emitter that does nothing
             self.emitter = EventEmitter()
         else:
             self.emitter = emitter
-        self.chunk_size = chunk_size
-        self.dry_run = dry_run
 
     def upload_stream(
         self,
         fp: T.IO[bytes],
-        cluster_filetype: upload_api_v4.ClusterFileType,
-        session_key: str,
+        session_key: str | None = None,
         progress: dict[str, T.Any] | None = None,
     ) -> str:
         if progress is None:
             progress = {}
 
+        if session_key is None:
+            session_key = self._gen_session_key(fp, progress)
+
         fp.seek(0, io.SEEK_END)
         entity_size = fp.tell()
 
-        upload_service = self._create_upload_service(session_key, cluster_filetype)
-
         progress["entity_size"] = entity_size
-        progress["chunk_size"] = self.chunk_size
+        progress["chunk_size"] = self.upload_options.chunk_size
         progress["retries"] = 0
         progress["begin_offset"] = None
 
@@ -372,10 +881,24 @@ class Uploader:
 
         while True:
             try:
-                file_handle = self._upload_stream_retryable(
-                    upload_service, fp, T.cast(UploaderProgress, progress)
-                )
-            except Exception as ex:
+                if self.user_session is not None:
+                    file_handle = self._upload_stream_retryable(
+                        self.user_session,
+                        fp,
+                        session_key,
+                        T.cast(UploaderProgress, progress),
+                    )
+                else:
+                    with api_v4.create_user_session(
+                        self.upload_options.user_items["user_upload_token"]
+                    ) as user_session:
+                        file_handle = self._upload_stream_retryable(
+                            user_session,
+                            fp,
+                            session_key,
+                            T.cast(UploaderProgress, progress),
+                        )
+            except BaseException as ex:  # Include KeyboardInterrupt
                 self._handle_upload_exception(ex, T.cast(UploaderProgress, progress))
             else:
                 break
@@ -384,75 +907,114 @@ class Uploader:
 
         self.emitter.emit("upload_end", progress)
 
-        # TODO: retry here
-        cluster_id = self._finish_upload_retryable(upload_service, file_handle)
-        progress["cluster_id"] = cluster_id
+        return file_handle
+
+    def finish_upload(
+        self,
+        file_handle: str,
+        cluster_filetype: api_v4.ClusterFileType,
+        progress: dict[str, T.Any] | None = None,
+    ) -> str:
+        """Finish upload with safe retries guraranteed"""
+        if progress is None:
+            progress = {}
+
+        if self.upload_options.dry_run or self.upload_options.nofinish:
+            cluster_id = "0"
+        else:
+            organization_id = self.upload_options.user_items.get("MAPOrganizationKey")
+
+            with api_v4.create_user_session(
+                self.upload_options.user_items["user_upload_token"]
+            ) as user_session:
+                resp = api_v4.finish_upload(
+                    user_session,
+                    file_handle,
+                    cluster_filetype,
+                    organization_id=organization_id,
+                )
 
+            body = api_v4.jsonify_response(resp)
+            # TODO: Validate cluster_id
+            cluster_id = body.get("cluster_id")
+
+        progress["cluster_id"] = cluster_id
         self.emitter.emit("upload_finished", progress)
 
         return cluster_id
 
     def _create_upload_service(
-        self, session_key: str, cluster_filetype: upload_api_v4.ClusterFileType
+        self, user_session: requests.Session, session_key: str
     ) -> upload_api_v4.UploadService:
         upload_service: upload_api_v4.UploadService
 
-        if self.dry_run:
+        if self.upload_options.dry_run:
+            upload_path = os.getenv("MAPILLARY_UPLOAD_ENDPOINT")
             upload_service = upload_api_v4.FakeUploadService(
-                user_access_token=self.user_items["user_upload_token"],
-                session_key=session_key,
-                cluster_filetype=cluster_filetype,
+                user_session,
+                session_key,
+                upload_path=Path(upload_path) if upload_path is not None else None,
             )
-        else:
-            upload_service = upload_api_v4.UploadService(
-                user_access_token=self.user_items["user_upload_token"],
-                session_key=session_key,
-                cluster_filetype=cluster_filetype,
+            LOG.info(
+                "Dry-run mode enabled, uploading to %s",
+                upload_service.upload_path.joinpath(session_key),
             )
+        else:
+            upload_service = upload_api_v4.UploadService(user_session, session_key)
 
         return upload_service
 
     def _handle_upload_exception(
-        self, ex: Exception, progress: UploaderProgress
+        self, ex: BaseException, progress: UploaderProgress
     ) -> None:
-        retries = progress["retries"]
+        retries = progress.get("retries", 0)
         begin_offset = progress.get("begin_offset")
-        chunk_size = progress["chunk_size"]
+        offset = progress.get("offset")
 
         if retries <= constants.MAX_UPLOAD_RETRIES and _is_retriable_exception(ex):
-            self.emitter.emit("upload_interrupted", progress)
+            self.emitter.emit("upload_retrying", progress)
+
             LOG.warning(
-                # use %s instead of %d because offset could be None
-                "Error uploading chunk_size %d at begin_offset %s: %s: %s",
-                chunk_size,
-                begin_offset,
-                ex.__class__.__name__,
-                str(ex),
+                f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
             )
+
             # Keep things immutable here. Will increment retries in the caller
             retries += 1
-            if _is_immediate_retry(ex):
+            if _is_immediate_retriable_exception(ex):
                 sleep_for = 0
             else:
                 sleep_for = min(2**retries, 16)
             LOG.info(
-                "Retrying in %d seconds (%d/%d)",
-                sleep_for,
-                retries,
-                constants.MAX_UPLOAD_RETRIES,
+                f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
             )
             if sleep_for:
                 time.sleep(sleep_for)
         else:
+            self.emitter.emit("upload_failed", progress)
             raise ex
 
+    @classmethod
+    def _upload_name(cls, progress: UploaderProgress):
+        # Strictly speaking these sequence properties should not be exposed in this context
+        # TODO: Maybe move these logging statements to event handlers
+        sequence_uuid: str | None = T.cast(
+            T.Union[str, None], progress.get("sequence_uuid")
+        )
+        import_path = T.cast(T.Union[str, None], progress.get("import_path"))
+        if sequence_uuid is not None:
+            if import_path is None:
+                name: str = f"sequence_{sequence_uuid}"
+            else:
+                name = f"sequence_{sequence_uuid}/{Path(import_path).name}"
+        else:
+            name = Path(import_path or "unknown").name
+        return name
+
     def _chunk_with_progress_emitted(
-        self,
-        stream: T.IO[bytes],
-        progress: UploaderProgress,
+        self, stream: T.IO[bytes], progress: UploaderProgress
     ) -> T.Generator[bytes, None, None]:
         for chunk in upload_api_v4.UploadService.chunkize_byte_stream(
-            stream, self.chunk_size
+            stream, self.upload_options.chunk_size
         ):
             yield chunk
 
@@ -465,11 +1027,21 @@ class Uploader:
 
     def _upload_stream_retryable(
         self,
-        upload_service: upload_api_v4.UploadService,
+        user_session: requests.Session,
         fp: T.IO[bytes],
-        progress: UploaderProgress,
+        session_key: str,
+        progress: UploaderProgress | None = None,
     ) -> str:
         """Upload the stream with safe retries guraranteed"""
+        if progress is None:
+            progress = T.cast(UploaderProgress, {})
+
+        upload_service = self._create_upload_service(user_session, session_key)
+
+        if "entity_size" not in progress:
+            fp.seek(0, io.SEEK_END)
+            entity_size = fp.tell()
+            progress["entity_size"] = entity_size
 
         begin_offset = upload_service.fetch_offset()
 
@@ -478,64 +1050,49 @@ class Uploader:
 
         self.emitter.emit("upload_fetch_offset", progress)
 
-        fp.seek(begin_offset, io.SEEK_SET)
+        # Estimate the read timeout
+        if not constants.MIN_UPLOAD_SPEED:
+            read_timeout = None
+        else:
+            remaining_bytes = abs(progress["entity_size"] - begin_offset)
+            read_timeout = max(
+                api_v4.REQUESTS_TIMEOUT,
+                remaining_bytes / constants.MIN_UPLOAD_SPEED,
+            )
 
+        # Upload from begin_offset
+        fp.seek(begin_offset, io.SEEK_SET)
         shifted_chunks = self._chunk_with_progress_emitted(fp, progress)
 
-        return upload_service.upload_shifted_chunks(shifted_chunks, begin_offset)
-
-    def _finish_upload_retryable(
-        self, upload_service: upload_api_v4.UploadService, file_handle: str
-    ) -> str:
-        """Finish upload with safe retries guraranteed"""
+        # Start uploading
+        return upload_service.upload_shifted_chunks(
+            shifted_chunks, begin_offset, read_timeout=read_timeout
+        )
 
-        if self.dry_run:
-            cluster_id = "0"
+    def _gen_session_key(self, fp: T.IO[bytes], progress: dict[str, T.Any]) -> str:
+        if self.upload_options.noresume:
+            # Generate a unique UUID for session_key when noresume is True
+            # to prevent resuming from previous uploads
+            session_key = f"{_prefixed_uuid4()}"
         else:
-            resp = api_v4.finish_upload(
-                self.user_items["user_upload_token"],
-                file_handle,
-                upload_service.cluster_filetype,
-                organization_id=self.user_items.get("MAPOrganizationKey"),
-            )
+            fp.seek(0, io.SEEK_SET)
+            session_key = utils.md5sum_fp(fp).hexdigest()
 
-            data = resp.json()
-            cluster_id = data.get("cluster_id")
+        filetype = progress.get("file_type")
+        if filetype is not None:
+            session_key = _suffix_session_key(session_key, types.FileType(filetype))
 
-            # TODO: validate cluster_id
-
-        return cluster_id
+        return session_key
 
 
 def _validate_metadatas(metadatas: T.Sequence[types.ImageMetadata]):
     for metadata in metadatas:
-        types.validate_image_desc(types.as_desc(metadata))
+        validate_image_desc(DescriptionJSONSerializer.as_desc(metadata))
         if not metadata.filename.is_file():
             raise FileNotFoundError(f"No such file {metadata.filename}")
 
 
-@contextmanager
-def wip_file_context(wip_path: Path, done_path: Path):
-    assert wip_path != done_path, "should not be the same file"
-    try:
-        os.remove(wip_path)
-    except FileNotFoundError:
-        pass
-    try:
-        yield wip_path
-        try:
-            os.remove(done_path)
-        except FileNotFoundError:
-            pass
-        wip_path.rename(done_path)
-    finally:
-        try:
-            os.remove(wip_path)
-        except FileNotFoundError:
-            pass
-
-
-def _is_immediate_retry(ex: Exception):
+def _is_immediate_retriable_exception(ex: BaseException) -> bool:
     if (
         isinstance(ex, requests.HTTPError)
         and isinstance(ex.response, requests.Response)
@@ -548,8 +1105,10 @@ def _is_immediate_retry(ex: Exception):
         # resp: {"debug_info":{"retriable":true,"type":"OffsetInvalidError","message":"Request starting offset is invalid"}}
         return resp.get("debug_info", {}).get("retriable", False)
 
+    return False
+
 
-def _is_retriable_exception(ex: Exception):
+def _is_retriable_exception(ex: BaseException) -> bool:
     if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
         return True
 
@@ -568,14 +1127,36 @@ def _is_retriable_exception(ex: Exception):
     return False
 
 
-_SUFFIX_MAP: dict[upload_api_v4.ClusterFileType, str] = {
-    upload_api_v4.ClusterFileType.ZIP: ".zip",
-    upload_api_v4.ClusterFileType.CAMM: ".mp4",
-    upload_api_v4.ClusterFileType.BLACKVUE: ".mp4",
+_SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
+    api_v4.ClusterFileType.ZIP: ".zip",
+    api_v4.ClusterFileType.CAMM: ".mp4",
+    api_v4.ClusterFileType.BLACKVUE: ".mp4",
+    types.FileType.IMAGE: ".jpg",
+    types.FileType.ZIP: ".zip",
+    types.FileType.BLACKVUE: ".mp4",
+    types.FileType.CAMM: ".mp4",
+    types.FileType.GOPRO: ".mp4",
+    types.FileType.VIDEO: ".mp4",
 }
 
 
-def _session_key(
-    upload_md5sum: str, cluster_filetype: upload_api_v4.ClusterFileType
+def _suffix_session_key(
+    key: str, filetype: api_v4.ClusterFileType | types.FileType
 ) -> str:
-    return f"mly_tools_{upload_md5sum}{_SUFFIX_MAP[cluster_filetype]}"
+    is_uuid_before = _is_uuid(key)
+
+    key = f"mly_tools_{key}{_SUFFIX_MAP[filetype]}"
+
+    assert _is_uuid(key) is is_uuid_before
+
+    return key
+
+
+def _prefixed_uuid4():
+    prefixed = f"uuid_{uuid.uuid4().hex}"
+    assert _is_uuid(prefixed)
+    return prefixed
+
+
+def _is_uuid(key: str) -> bool:
+    return key.startswith("uuid_") or key.startswith("mly_tools_uuid_")