mapillary-tools 0.14.0b1__py3-none-any.whl → 0.14.2__py3-none-any.whl

mapillary_tools/uploader.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import concurrent.futures
-
 import dataclasses
 import io
 import json
 import logging
 import os
+import queue
 import struct
 import sys
 import tempfile
@@ -25,7 +25,22 @@ else:
 
 import requests
 
-from . import api_v4, config, constants, exif_write, types, upload_api_v4, utils
+from . import (
+    api_v4,
+    config,
+    constants,
+    exif_write,
+    geo,
+    history,
+    telemetry,
+    types,
+    upload_api_v4,
+    utils,
+    VERSION,
+)
+from .camm import camm_builder, camm_parser
+from .gpmf import gpmf_parser
+from .mp4 import simple_mp4_builder
 from .serializer.description import (
     desc_file_to_exif,
     DescriptionJSONSerializer,
@@ -36,6 +51,25 @@ from .serializer.description import (
 LOG = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass(frozen=True)
+class UploadOptions:
+    user_items: config.UserItem
+    chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024)
+    num_upload_workers: int = constants.MAX_IMAGE_UPLOAD_WORKERS
+    dry_run: bool = False
+    nofinish: bool = False
+    noresume: bool = False
+
+    def __post_init__(self):
+        if self.num_upload_workers <= 0:
+            raise ValueError(
+                f"Expect positive num_upload_workers but got {self.num_upload_workers}"
+            )
+
+        if self.chunk_size <= 0:
+            raise ValueError(f"Expect positive chunk_size but got {self.chunk_size}")
+
+
 class UploaderProgress(T.TypedDict, total=True):
     """
     Progress data that Uploader cares about.
@@ -60,7 +94,7 @@ class UploaderProgress(T.TypedDict, total=True):
     #   - offset == entity_size when "upload_end" or "upload_finished"
     entity_size: int
 
-    # An "upload_interrupted" will increase it. Reset to 0 if a chunk is uploaded
+    # An "upload_retrying" will increase it. Reset to 0 if a chunk is uploaded
     retries: int
 
     # Cluster ID after finishing the upload
@@ -92,7 +126,7 @@ class SequenceProgress(T.TypedDict, total=False):
     # MAPSequenceUUID. It is only available for directory uploading
     sequence_uuid: str
 
-    # Path to the Zipfile/BlackVue/CAMM
+    # Path to the image/video/zip
     import_path: str
 
 
@@ -120,13 +154,43 @@ class InvalidMapillaryZipFileError(SequenceError):
     pass
 
 
+# BELOW demonstrates the pseudocode for a typical upload workflow
+# and when upload events are emitted
+#################################################################
+# def pseudo_upload(metadata):
+#     emit("upload_start")
+#     while True:
+#         try:
+#             if is_sequence(metadata):
+#                 for image in metadata:
+#                     upload_stream(image.read())
+#                     emit("upload_progress")
+#             elif is_video(metadata):
+#                 offset = fetch_offset()
+#                 emit("upload_fetch_offset")
+#                 for chunk in metadata.read()[offset:]:
+#                     upload_stream(chunk)
+#                     emit("upload_progress")
+#         except BaseException as ex:  # Include KeyboardInterrupt
+#             if retryable(ex):
+#                 emit("upload_retrying")
+#                 continue
+#             else:
+#                 emit("upload_failed")
+#                 raise ex
+#         else:
+#             break
+#     emit("upload_end")
+#     finish_upload(data)
+#     emit("upload_finished")
 EventName = T.Literal[
     "upload_start",
     "upload_fetch_offset",
     "upload_progress",
+    "upload_retrying",
     "upload_end",
+    "upload_failed",
     "upload_finished",
-    "upload_interrupted",
 ]
 
 
@@ -154,7 +218,131 @@ class UploadResult:
     error: Exception | None = None
 
 
-class ZipImageSequence:
+class VideoUploader:
+    @classmethod
+    def upload_videos(
+        cls, mly_uploader: Uploader, video_metadatas: T.Sequence[types.VideoMetadata]
+    ) -> T.Generator[tuple[types.VideoMetadata, UploadResult], None, None]:
+        # If upload in a random order, then interrupted uploads has a higher chance to expire.
+        # Therefore sort videos to make sure interrupted uploads are resumed as early as possible
+        sorted_video_metadatas = sorted(video_metadatas, key=lambda m: m.filename)
+
+        for idx, video_metadata in enumerate(sorted_video_metadatas):
+            LOG.debug(f"Checksum for video {video_metadata.filename}...")
+            try:
+                video_metadata.update_md5sum()
+            except Exception as ex:
+                yield video_metadata, UploadResult(error=ex)
+                continue
+
+            assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
+
+            progress: SequenceProgress = {
+                "total_sequence_count": len(sorted_video_metadatas),
+                "sequence_idx": idx,
+                "file_type": video_metadata.filetype.value,
+                "import_path": str(video_metadata.filename),
+                "sequence_md5sum": video_metadata.md5sum,
+            }
+
+            try:
+                with cls.build_camm_stream(video_metadata) as camm_fp:
+                    # Upload the mp4 stream
+                    file_handle = mly_uploader.upload_stream(
+                        T.cast(T.IO[bytes], camm_fp),
+                        progress=T.cast(T.Dict[str, T.Any], progress),
+                    )
+
+                cluster_id = mly_uploader.finish_upload(
+                    file_handle,
+                    api_v4.ClusterFileType.CAMM,
+                    progress=T.cast(T.Dict[str, T.Any], progress),
+                )
+            except Exception as ex:
+                yield video_metadata, UploadResult(error=ex)
+            else:
+                yield video_metadata, UploadResult(result=cluster_id)
+
+    @classmethod
+    @contextmanager
+    def build_camm_stream(cls, video_metadata: types.VideoMetadata):
+        # Convert video metadata to CAMMInfo
+        camm_info = cls.prepare_camm_info(video_metadata)
+
+        # Create the CAMM sample generator
+        camm_sample_generator = camm_builder.camm_sample_generator2(camm_info)
+
+        with video_metadata.filename.open("rb") as src_fp:
+            # Build the mp4 stream with the CAMM samples
+            yield simple_mp4_builder.transform_mp4(src_fp, camm_sample_generator)
+
+    @classmethod
+    def prepare_camm_info(
+        cls, video_metadata: types.VideoMetadata
+    ) -> camm_parser.CAMMInfo:
+        camm_info = camm_parser.CAMMInfo(
+            make=video_metadata.make or "", model=video_metadata.model or ""
+        )
+
+        for point in video_metadata.points:
+            if isinstance(point, telemetry.CAMMGPSPoint):
+                if camm_info.gps is None:
+                    camm_info.gps = []
+                camm_info.gps.append(point)
+
+            elif isinstance(point, telemetry.GPSPoint):
+                # There is no proper CAMM entry for GoPro GPS
+                if camm_info.mini_gps is None:
+                    camm_info.mini_gps = []
+                camm_info.mini_gps.append(point)
+
+            elif isinstance(point, geo.Point):
+                if camm_info.mini_gps is None:
+                    camm_info.mini_gps = []
+                camm_info.mini_gps.append(point)
+            else:
+                raise ValueError(f"Unknown point type: {point}")
+
+        if constants.MAPILLARY__EXPERIMENTAL_ENABLE_IMU:
+            if video_metadata.filetype is types.FileType.GOPRO:
+                with video_metadata.filename.open("rb") as fp:
+                    gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True)
+                if gopro_info is not None:
+                    camm_info.accl = gopro_info.accl or []
+                    camm_info.gyro = gopro_info.gyro or []
+                    camm_info.magn = gopro_info.magn or []
+
+        return camm_info
+
+
+class ZipUploader:
+    @classmethod
+    def upload_zipfiles(
+        cls, mly_uploader: Uploader, zip_paths: T.Sequence[Path]
+    ) -> T.Generator[tuple[Path, UploadResult], None, None]:
+        # If upload in a random order, then interrupted uploads has a higher chance to expire.
+        # Therefore sort zipfiles to make sure interrupted uploads are resumed as early as possible
+        sorted_zip_paths = sorted(zip_paths)
+
+        for idx, zip_path in enumerate(sorted_zip_paths):
+            progress: SequenceProgress = {
+                "total_sequence_count": len(sorted_zip_paths),
+                "sequence_idx": idx,
+                "import_path": str(zip_path),
+                "file_type": types.FileType.ZIP.value,
+                "sequence_md5sum": "",  # Placeholder, will be set in upload_zipfile
+            }
+            try:
+                cluster_id = cls._upload_zipfile(
+                    mly_uploader,
+                    zip_path,
+                    progress=T.cast(T.Dict[str, T.Any], progress),
+                )
+            except Exception as ex:
+                yield zip_path, UploadResult(error=ex)
+            else:
+                yield zip_path, UploadResult(result=cluster_id)
+
     @classmethod
     def zip_images(
         cls, metadatas: T.Sequence[types.ImageMetadata], zip_dir: Path
@@ -173,38 +361,93 @@ class ZipImageSequence:
             )
             with cls._wip_file_context(wip_zip_filename) as wip_path:
                 with wip_path.open("wb") as wip_fp:
-                    cls.zip_sequence_fp(sequence, wip_fp)
+                    cls._zip_sequence_fp(sequence, wip_fp)
 
     @classmethod
-    @contextmanager
-    def _wip_file_context(cls, wip_path: Path):
-        try:
-            os.remove(wip_path)
-        except FileNotFoundError:
-            pass
-        try:
-            yield wip_path
+    def zip_images_and_upload(
+        cls, uploader: Uploader, image_metadatas: T.Sequence[types.ImageMetadata]
+    ) -> T.Generator[tuple[str, UploadResult], None, None]:
+        sequences = types.group_and_sort_images(image_metadatas)
 
-            with wip_path.open("rb") as fp:
-                upload_md5sum = utils.md5sum_fp(fp).hexdigest()
+        for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
+            try:
+                _validate_metadatas(sequence)
+            except Exception as ex:
+                yield sequence_uuid, UploadResult(error=ex)
+                continue
 
-            done_path = wip_path.parent.joinpath(
-                _session_key(upload_md5sum, api_v4.ClusterFileType.ZIP)
+            with tempfile.NamedTemporaryFile() as fp:
+                try:
+                    sequence_md5sum = cls._zip_sequence_fp(sequence, fp)
+                except Exception as ex:
+                    yield sequence_uuid, UploadResult(error=ex)
+                    continue
+
+                sequence_progress: SequenceProgress = {
+                    "sequence_idx": sequence_idx,
+                    "total_sequence_count": len(sequences),
+                    "sequence_image_count": len(sequence),
+                    "sequence_uuid": sequence_uuid,
+                    "file_type": types.FileType.ZIP.value,
+                    "sequence_md5sum": sequence_md5sum,
+                }
+
+                try:
+                    file_handle = uploader.upload_stream(
+                        fp, progress=T.cast(T.Dict[str, T.Any], sequence_progress)
+                    )
+                    cluster_id = uploader.finish_upload(
+                        file_handle,
+                        api_v4.ClusterFileType.ZIP,
+                        progress=T.cast(T.Dict[str, T.Any], sequence_progress),
+                    )
+                except Exception as ex:
+                    yield sequence_uuid, UploadResult(error=ex)
+                    continue
+
+            yield sequence_uuid, UploadResult(result=cluster_id)
+
+    @classmethod
+    def _upload_zipfile(
+        cls,
+        uploader: Uploader,
+        zip_path: Path,
+        progress: dict[str, T.Any] | None = None,
+    ) -> str:
+        if progress is None:
+            progress = {}
+
+        with zipfile.ZipFile(zip_path) as ziph:
+            namelist = ziph.namelist()
+            if not namelist:
+                raise InvalidMapillaryZipFileError("Zipfile has no files")
+
+        with zip_path.open("rb") as zip_fp:
+            sequence_md5sum = cls._extract_sequence_md5sum(zip_fp)
+
+        # Send the copy of the input progress to each upload session, to avoid modifying the original one
+        mutable_progress: SequenceProgress = {
+            **T.cast(SequenceProgress, progress),
+            "sequence_image_count": len(namelist),
+            "sequence_md5sum": sequence_md5sum,
+            "file_type": types.FileType.ZIP.value,
+        }
+
+        with zip_path.open("rb") as zip_fp:
+            file_handle = uploader.upload_stream(
+                zip_fp, progress=T.cast(T.Dict[str, T.Any], mutable_progress)
             )
 
-            try:
-                os.remove(done_path)
-            except FileNotFoundError:
-                pass
-            wip_path.rename(done_path)
-        finally:
-            try:
-                os.remove(wip_path)
-            except FileNotFoundError:
-                pass
+        cluster_id = uploader.finish_upload(
+            file_handle,
+            api_v4.ClusterFileType.ZIP,
+            progress=T.cast(T.Dict[str, T.Any], mutable_progress),
+        )
+
+        return cluster_id
 
     @classmethod
-    def zip_sequence_fp(
+    def _zip_sequence_fp(
         cls,
         sequence: T.Sequence[types.ImageMetadata],
         zip_fp: T.IO[bytes],
@@ -219,6 +462,8 @@ class ZipImageSequence:
             f"Only one sequence is allowed but got {len(sequence_groups)}: {list(sequence_groups.keys())}"
         )
 
+        if sequence:
+            LOG.debug(f"Checksum for sequence {sequence[0].MAPSequenceUUID}...")
         sequence_md5sum = types.update_sequence_md5sum(sequence)
 
         with zipfile.ZipFile(zip_fp, "w", zipfile.ZIP_DEFLATED) as zipf:
@@ -226,16 +471,18 @@ class ZipImageSequence:
                 # Arcname should be unique, the name does not matter
                 arcname = f"{idx}.jpg"
                 zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
-                zipf.writestr(zipinfo, cls._dump_image_bytes(metadata))
+                zipf.writestr(zipinfo, SingleImageUploader.dump_image_bytes(metadata))
             assert len(sequence) == len(set(zipf.namelist()))
-            zipf.comment = json.dumps({"sequence_md5sum": sequence_md5sum}).encode(
-                "utf-8"
-            )
+            zipf.comment = json.dumps(
+                {"sequence_md5sum": sequence_md5sum},
+                sort_keys=True,
+                separators=(",", ":"),
+            ).encode("utf-8")
 
         return sequence_md5sum
 
     @classmethod
-    def extract_sequence_md5sum(cls, zip_fp: T.IO[bytes]) -> str:
+    def _extract_sequence_md5sum(cls, zip_fp: T.IO[bytes]) -> str:
         with zipfile.ZipFile(zip_fp, "r", zipfile.ZIP_DEFLATED) as ziph:
             comment = ziph.comment
 
@@ -258,237 +505,365 @@ class ZipImageSequence:
         return sequence_md5sum
 
     @classmethod
-    def _dump_image_bytes(cls, metadata: types.ImageMetadata) -> bytes:
+    @contextmanager
+    def _wip_file_context(cls, wip_path: Path):
         try:
-            edit = exif_write.ExifEdit(metadata.filename)
-        except struct.error as ex:
-            raise ExifError(f"Failed to load EXIF: {ex}", metadata.filename) from ex
+            os.remove(wip_path)
+        except FileNotFoundError:
+            pass
+        try:
+            yield wip_path
 
-        # The cast is to fix the type checker error
-        edit.add_image_description(
-            T.cast(
-                T.Dict,
-                desc_file_to_exif(DescriptionJSONSerializer.as_desc(metadata)),
+            with wip_path.open("rb") as fp:
+                upload_md5sum = utils.md5sum_fp(fp).hexdigest()
+
+            done_path = wip_path.parent.joinpath(
+                _suffix_session_key(upload_md5sum, api_v4.ClusterFileType.ZIP)
             )
-        )
 
-        try:
-            return edit.dump_image_bytes()
-        except struct.error as ex:
-            raise ExifError(
-                f"Failed to dump EXIF bytes: {ex}", metadata.filename
-            ) from ex
+            try:
+                os.remove(done_path)
+            except FileNotFoundError:
+                pass
+            wip_path.rename(done_path)
+        finally:
+            try:
+                os.remove(wip_path)
+            except FileNotFoundError:
+                pass
 
-    @classmethod
-    def upload_zipfile(
-        cls,
-        uploader: Uploader,
-        zip_path: Path,
-        progress: dict[str, T.Any] | None = None,
-    ) -> str:
-        if progress is None:
-            progress = {}
 
-        with zipfile.ZipFile(zip_path) as ziph:
-            namelist = ziph.namelist()
-            if not namelist:
-                raise InvalidMapillaryZipFileError("Zipfile has no files")
+class ImageSequenceUploader:
+    def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
+        self.upload_options = upload_options
+        self.emitter = emitter
 
-        with zip_path.open("rb") as zip_fp:
-            sequence_md5sum = cls.extract_sequence_md5sum(zip_fp)
+    def upload_images(
+        self, image_metadatas: T.Sequence[types.ImageMetadata]
+    ) -> T.Generator[tuple[str, UploadResult], None, None]:
+        sequences = types.group_and_sort_images(image_metadatas)
 
-        # Send the copy of the input progress to each upload session, to avoid modifying the original one
-        mutable_progress: SequenceProgress = {
-            **T.cast(SequenceProgress, progress),
-            "sequence_image_count": len(namelist),
-            "sequence_md5sum": sequence_md5sum,
-            "file_type": types.FileType.ZIP.value,
-        }
+        for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
+            LOG.debug(f"Checksum for image sequence {sequence_uuid}...")
+            sequence_md5sum = types.update_sequence_md5sum(sequence)
 
-        with zip_path.open("rb") as zip_fp:
-            file_handle = uploader.upload_stream(
-                zip_fp, progress=T.cast(T.Dict[str, T.Any], mutable_progress)
+            sequence_progress: SequenceProgress = {
+                "sequence_idx": sequence_idx,
+                "total_sequence_count": len(sequences),
+                "sequence_image_count": len(sequence),
+                "sequence_uuid": sequence_uuid,
+                "file_type": types.FileType.IMAGE.value,
+                "sequence_md5sum": sequence_md5sum,
+            }
+
+            try:
+                cluster_id = self._upload_sequence_and_finish(
+                    sequence,
+                    sequence_progress=T.cast(dict[str, T.Any], sequence_progress),
+                )
+            except Exception as ex:
+                yield sequence_uuid, UploadResult(error=ex)
+            else:
+                yield sequence_uuid, UploadResult(result=cluster_id)
+
+    def _upload_sequence_and_finish(
+        self,
+        sequence: T.Sequence[types.ImageMetadata],
+        sequence_progress: dict[str, T.Any],
+    ) -> str:
+        _validate_metadatas(sequence)
+
+        sequence_progress["entity_size"] = sum(m.filesize or 0 for m in sequence)
+        self.emitter.emit("upload_start", sequence_progress)
+
+        try:
+            # Retries will be handled in the call (but no upload event emissions)
+            image_file_handles = self._upload_images_parallel(
+                sequence, sequence_progress
             )
+        except BaseException as ex:  # Include KeyboardInterrupt
+            self.emitter.emit("upload_failed", sequence_progress)
+            raise ex
 
+        manifest_file_handle = self._upload_manifest(image_file_handles)
+
+        self.emitter.emit("upload_end", sequence_progress)
+
+        uploader = Uploader(self.upload_options, emitter=self.emitter)
         cluster_id = uploader.finish_upload(
-            file_handle,
-            api_v4.ClusterFileType.ZIP,
-            progress=T.cast(T.Dict[str, T.Any], mutable_progress),
+            manifest_file_handle,
+            api_v4.ClusterFileType.MLY_BUNDLE_MANIFEST,
+            progress=sequence_progress,
         )
 
         return cluster_id
 
-    @classmethod
-    def zip_images_and_upload(
-        cls,
-        uploader: Uploader,
-        image_metadatas: T.Sequence[types.ImageMetadata],
-        progress: dict[str, T.Any] | None = None,
-    ) -> T.Generator[tuple[str, UploadResult], None, None]:
-        if progress is None:
-            progress = {}
+    def _upload_manifest(self, image_file_handles: T.Sequence[str]) -> str:
+        uploader = Uploader(self.upload_options)
 
-        sequences = types.group_and_sort_images(image_metadatas)
+        manifest = {
+            "version": "1",
+            "upload_type": "images",
+            "image_handles": image_file_handles,
+        }
 
-        for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
-            try:
-                _validate_metadatas(sequence)
-            except Exception as ex:
-                yield sequence_uuid, UploadResult(error=ex)
-                continue
+        with io.BytesIO() as manifest_fp:
+            manifest_fp.write(
+                json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode(
+                    "utf-8"
+                )
+            )
+            manifest_fp.seek(0, io.SEEK_SET)
+            return uploader.upload_stream(
+                manifest_fp, session_key=f"{_prefixed_uuid4()}.json"
+            )
 
-            with tempfile.NamedTemporaryFile() as fp:
-                try:
-                    sequence_md5sum = cls.zip_sequence_fp(sequence, fp)
-                except Exception as ex:
-                    yield sequence_uuid, UploadResult(error=ex)
-                    continue
+    def _upload_images_parallel(
+        self,
+        sequence: T.Sequence[types.ImageMetadata],
+        sequence_progress: dict[str, T.Any],
+    ) -> list[str]:
+        if not sequence:
+            return []
 
-                sequence_progress: SequenceProgress = {
-                    "sequence_idx": sequence_idx,
-                    "total_sequence_count": len(sequences),
-                    "sequence_image_count": len(sequence),
-                    "sequence_uuid": sequence_uuid,
-                    "file_type": types.FileType.ZIP.value,
-                    "sequence_md5sum": sequence_md5sum,
-                }
+        max_workers = min(self.upload_options.num_upload_workers, len(sequence))
 
-                mutable_progress: dict[str, T.Any] = {**progress, **sequence_progress}
+        # Lock is used to synchronize event emission
+        lock = threading.Lock()
 
-                try:
-                    file_handle = uploader.upload_stream(fp, progress=mutable_progress)
-                    cluster_id = uploader.finish_upload(
-                        file_handle,
-                        api_v4.ClusterFileType.ZIP,
-                        progress=mutable_progress,
-                    )
-                except Exception as ex:
-                    yield sequence_uuid, UploadResult(error=ex)
-                    continue
+        # Push all images into the queue
+        image_queue: queue.Queue[tuple[int, types.ImageMetadata]] = queue.Queue()
+        for idx, image_metadata in enumerate(sequence):
+            image_queue.put((idx, image_metadata))
+
+        upload_interrupted = threading.Event()
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(
+                    self._upload_images_from_queue,
+                    image_queue,
+                    lock,
+                    upload_interrupted,
+                    sequence_progress,
+                )
+                for _ in range(max_workers)
+            ]
 
-            yield sequence_uuid, UploadResult(result=cluster_id)
+            indexed_image_file_handles = []
 
-    @classmethod
-    def _upload_sequence(
-        cls,
-        uploader: Uploader,
-        sequence: T.Sequence[types.ImageMetadata],
-        progress: dict[str, T.Any] | None = None,
-    ) -> str:
-        if progress is None:
-            progress = {}
+            try:
+                for future in futures:
+                    indexed_image_file_handles.extend(future.result())
+            except KeyboardInterrupt as ex:
+                upload_interrupted.set()
+                raise ex
 
-        # FIXME: This is a hack to disable the event emitter inside the uploader
-        uploader_without_emitter = uploader.copy_uploader_without_emitter()
+        # All tasks should be done here, so below is more like assertion
+        image_queue.join()
+        if sys.version_info >= (3, 13):
+            image_queue.shutdown()
 
-        lock = threading.Lock()
+        file_handles: list[str] = []
 
-        def _upload_image(image_metadata: types.ImageMetadata) -> str:
-            mutable_progress = {
-                **(progress or {}),
-                "filename": str(image_metadata.filename),
-            }
+        indexed_image_file_handles.sort()
 
-            bytes = cls._dump_image_bytes(image_metadata)
-            file_handle = uploader_without_emitter.upload_stream(
-                io.BytesIO(bytes), progress=mutable_progress
+        # Important to guarantee the order
+        assert len(indexed_image_file_handles) == len(sequence)
+        for expected_idx, (idx, file_handle) in enumerate(indexed_image_file_handles):
+            assert expected_idx == idx
+            file_handles.append(file_handle)
+
+        return file_handles
+
+    def _upload_images_from_queue(
+        self,
+        image_queue: queue.Queue[tuple[int, types.ImageMetadata]],
+        lock: threading.Lock,
+        upload_interrupted: threading.Event,
+        sequence_progress: dict[str, T.Any],
+    ) -> list[tuple[int, str]]:
+        indexed_file_handles = []
+
+        with api_v4.create_user_session(
+            self.upload_options.user_items["user_upload_token"]
+        ) as user_session:
+            single_image_uploader = SingleImageUploader(
+                self.upload_options, user_session=user_session
             )
 
-            mutable_progress["chunk_size"] = image_metadata.filesize
+            while True:
+                # Assert that all images are already pushed into the queue
+                try:
+                    idx, image_metadata = image_queue.get_nowait()
+                except queue.Empty:
+                    break
+
+                # Main thread will handle the interruption
+                if upload_interrupted.is_set():
+                    break
+
+                # Create a new mutatble progress to keep the sequence_progress immutable
+                image_progress = {
+                    **sequence_progress,
+                    "import_path": str(image_metadata.filename),
+                }
 
-            with lock:
-                uploader.emitter.emit("upload_progress", mutable_progress)
+                # image_progress will be updated during uploading
+                file_handle = single_image_uploader.upload(
+                    image_metadata, image_progress
+                )
 
-            return file_handle
+                # Update chunk_size (it was constant if set)
+                image_progress["chunk_size"] = image_metadata.filesize
 
-        _validate_metadatas(sequence)
+                # Main thread will handle the interruption
+                if upload_interrupted.is_set():
+                    break
 
-        progress["entity_size"] = sum(m.filesize or 0 for m in sequence)
+                with lock:
+                    self.emitter.emit("upload_progress", image_progress)
 
-        # TODO: assert sequence is sorted
+                indexed_file_handles.append((idx, file_handle))
 
-        uploader.emitter.emit("upload_start", progress)
+                image_queue.task_done()
 
-        with concurrent.futures.ThreadPoolExecutor(
-            max_workers=constants.MAX_IMAGE_UPLOAD_WORKERS
-        ) as executor:
-            image_file_handles = list(executor.map(_upload_image, sequence))
+        return indexed_file_handles
 
-        manifest = {
-            "version": "1",
-            "upload_type": "images",
-            "image_handles": image_file_handles,
-        }
 
-        with io.BytesIO() as manifest_fp:
-            manifest_fp.write(json.dumps(manifest).encode("utf-8"))
-            manifest_fp.seek(0, io.SEEK_SET)
-            manifest_file_handle = uploader_without_emitter.upload_stream(
-                manifest_fp, session_key=f"{uuid.uuid4().hex}.json"
+class SingleImageUploader:
+    def __init__(
+        self,
+        upload_options: UploadOptions,
+        user_session: requests.Session | None = None,
+    ):
+        self.upload_options = upload_options
+        self.user_session = user_session
+        self.cache = self._maybe_create_persistent_cache_instance(
+            self.upload_options.user_items, upload_options
+        )
+
+    def upload(
+        self, image_metadata: types.ImageMetadata, image_progress: dict[str, T.Any]
+    ) -> str:
+        image_bytes = self.dump_image_bytes(image_metadata)
+
+        uploader = Uploader(self.upload_options, user_session=self.user_session)
+
+        session_key = uploader._gen_session_key(io.BytesIO(image_bytes), image_progress)
+
+        file_handle = self._get_cached_file_handle(session_key)
+
+        if file_handle is None:
+            # image_progress will be updated during uploading
+            file_handle = uploader.upload_stream(
+                io.BytesIO(image_bytes),
+                session_key=session_key,
+                progress=image_progress,
             )
+            self._set_file_handle_cache(session_key, file_handle)
 
-        uploader.emitter.emit("upload_end", progress)
+        return file_handle
 
-        cluster_id = uploader.finish_upload(
-            manifest_file_handle,
-            api_v4.ClusterFileType.MLY_BUNDLE_MANIFEST,
-            progress=progress,
+    @classmethod
+    def dump_image_bytes(cls, metadata: types.ImageMetadata) -> bytes:
+        try:
+            edit = exif_write.ExifEdit(metadata.filename)
+        except struct.error as ex:
+            raise ExifError(f"Failed to load EXIF: {ex}", metadata.filename) from ex
+
+        # The cast is to fix the type checker error
+        edit.add_image_description(
+            T.cast(
+                T.Dict, desc_file_to_exif(DescriptionJSONSerializer.as_desc(metadata))
+            )
         )
 
-        return cluster_id
+        try:
+            return edit.dump_image_bytes()
+        except struct.error as ex:
+            raise ExifError(
+                f"Failed to dump EXIF bytes: {ex}", metadata.filename
+            ) from ex
 
     @classmethod
-    def upload_images(
-        cls,
-        uploader: Uploader,
-        image_metadatas: T.Sequence[types.ImageMetadata],
-        progress: dict[str, T.Any] | None = None,
-    ) -> T.Generator[tuple[str, UploadResult], None, None]:
-        if progress is None:
-            progress = {}
+    def _maybe_create_persistent_cache_instance(
+        cls, user_items: config.UserItem, upload_options: UploadOptions
+    ) -> history.PersistentCache | None:
+        if not constants.UPLOAD_CACHE_DIR:
+            LOG.debug(
+                "Upload cache directory is set empty, skipping caching upload file handles"
+            )
+            return None
+
+        if upload_options.dry_run:
+            LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
+            return None
+
+        # Different python/CLI versions use different cache (dbm) formats.
+        # Separate them to avoid conflicts
+        py_version_parts = [str(part) for part in sys.version_info[:3]]
+        version = f"py_{'_'.join(py_version_parts)}_{VERSION}"
+
+        cache_path_dir = (
+            Path(constants.UPLOAD_CACHE_DIR)
+            .joinpath(version)
+            .joinpath(api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"))
+            .joinpath(
+                user_items.get("MAPSettingsUserKey", user_items["user_upload_token"])
+            )
+        )
+        cache_path_dir.mkdir(parents=True, exist_ok=True)
+        cache_path = cache_path_dir.joinpath("cached_file_handles")
+
+        # Sanitize sensitive segments for logging
+        sanitized_cache_path = (
+            Path(constants.UPLOAD_CACHE_DIR)
+            .joinpath(version)
+            .joinpath("***")
+            .joinpath("***")
+            .joinpath("cached_file_handles")
+        )
+        LOG.debug(f"File handle cache path: {sanitized_cache_path}")
 
-        sequences = types.group_and_sort_images(image_metadatas)
+        cache = history.PersistentCache(str(cache_path.resolve()))
+        cache.clear_expired()
 
-        for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
-            sequence_md5sum = types.update_sequence_md5sum(sequence)
+        return cache
 
-            sequence_progress: SequenceProgress = {
-                "sequence_idx": sequence_idx,
-                "total_sequence_count": len(sequences),
-                "sequence_image_count": len(sequence),
-                "sequence_uuid": sequence_uuid,
-                "file_type": types.FileType.IMAGE.value,
-                "sequence_md5sum": sequence_md5sum,
-            }
+    def _get_cached_file_handle(self, key: str) -> str | None:
+        if self.cache is None:
+            return None
 
-            mutable_progress: dict[str, T.Any] = {**progress, **sequence_progress}
+        if _is_uuid(key):
+            return None
 
-            try:
-                cluster_id = cls._upload_sequence(
-                    uploader, sequence, progress=mutable_progress
-                )
-            except Exception as ex:
-                yield sequence_uuid, UploadResult(error=ex)
-            else:
-                yield sequence_uuid, UploadResult(result=cluster_id)
+        return self.cache.get(key)
+
+    def _set_file_handle_cache(self, key: str, value: str) -> None:
+        if self.cache is None:
+            return
+
+        if _is_uuid(key):
+            return
+
+        self.cache.set(key, value)
 
 
 class Uploader:
     def __init__(
         self,
-        user_items: config.UserItem,
+        upload_options: UploadOptions,
+        user_session: requests.Session | None = None,
         emitter: EventEmitter | None = None,
-        chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
-        dry_run=False,
     ):
-        self.user_items = user_items
+        self.upload_options = upload_options
+        self.user_session = user_session
         if emitter is None:
             # An empty event emitter that does nothing
             self.emitter = EventEmitter()
         else:
             self.emitter = emitter
-        self.chunk_size = chunk_size
-        self.dry_run = dry_run
 
     def upload_stream(
         self,
@@ -500,21 +875,13 @@ class Uploader:
             progress = {}
 
         if session_key is None:
-            fp.seek(0, io.SEEK_SET)
-            md5sum = utils.md5sum_fp(fp).hexdigest()
-            filetype = progress.get("file_type")
-            if filetype is not None:
-                session_key = _session_key(md5sum, types.FileType(filetype))
-            else:
-                session_key = md5sum
+            session_key = self._gen_session_key(fp, progress)
 
         fp.seek(0, io.SEEK_END)
         entity_size = fp.tell()
 
-        upload_service = self._create_upload_service(session_key)
-
         progress["entity_size"] = entity_size
-        progress["chunk_size"] = self.chunk_size
+        progress["chunk_size"] = self.upload_options.chunk_size
         progress["retries"] = 0
         progress["begin_offset"] = None
 
@@ -522,10 +889,24 @@ class Uploader:
 
         while True:
             try:
-                file_handle = self._upload_stream_retryable(
-                    upload_service, fp, T.cast(UploaderProgress, progress)
-                )
-            except Exception as ex:
+                if self.user_session is not None:
+                    file_handle = self._upload_stream_retryable(
+                        self.user_session,
+                        fp,
+                        session_key,
+                        T.cast(UploaderProgress, progress),
+                    )
+                else:
+                    with api_v4.create_user_session(
+                        self.upload_options.user_items["user_upload_token"]
+                    ) as user_session:
+                        file_handle = self._upload_stream_retryable(
+                            user_session,
+                            fp,
+                            session_key,
+                            T.cast(UploaderProgress, progress),
+                        )
+            except BaseException as ex:  # Include KeyboardInterrupt
                 self._handle_upload_exception(ex, T.cast(UploaderProgress, progress))
             else:
                 break
@@ -546,97 +927,102 @@ class Uploader:
         if progress is None:
             progress = {}
 
-        if self.dry_run:
+        if self.upload_options.dry_run or self.upload_options.nofinish:
             cluster_id = "0"
         else:
-            resp = api_v4.finish_upload(
-                self.user_items["user_upload_token"],
-                file_handle,
-                cluster_filetype,
-                organization_id=self.user_items.get("MAPOrganizationKey"),
-            )
-
-            data = resp.json()
-            cluster_id = data.get("cluster_id")
+            organization_id = self.upload_options.user_items.get("MAPOrganizationKey")
+
+            with api_v4.create_user_session(
+                self.upload_options.user_items["user_upload_token"]
+            ) as user_session:
+                resp = api_v4.finish_upload(
+                    user_session,
+                    file_handle,
+                    cluster_filetype,
+                    organization_id=organization_id,
+                )
 
-            # TODO: validate cluster_id
+            body = api_v4.jsonify_response(resp)
+            # TODO: Validate cluster_id
+            cluster_id = body.get("cluster_id")
 
         progress["cluster_id"] = cluster_id
         self.emitter.emit("upload_finished", progress)
 
         return cluster_id
 
-    def copy_uploader_without_emitter(self) -> Uploader:
-        return Uploader(
-            self.user_items,
-            emitter=None,
-            chunk_size=self.chunk_size,
-            dry_run=self.dry_run,
-        )
-
-    def _create_upload_service(self, session_key: str) -> upload_api_v4.UploadService:
+    def _create_upload_service(
+        self, user_session: requests.Session, session_key: str
+    ) -> upload_api_v4.UploadService:
         upload_service: upload_api_v4.UploadService
 
-        if self.dry_run:
+        if self.upload_options.dry_run:
             upload_path = os.getenv("MAPILLARY_UPLOAD_ENDPOINT")
             upload_service = upload_api_v4.FakeUploadService(
-                user_access_token=self.user_items["user_upload_token"],
-                session_key=session_key,
+                user_session,
+                session_key,
                 upload_path=Path(upload_path) if upload_path is not None else None,
             )
             LOG.info(
-                "Dry run mode enabled. Data will be uploaded to %s",
+                "Dry-run mode enabled, uploading to %s",
                 upload_service.upload_path.joinpath(session_key),
             )
         else:
-            upload_service = upload_api_v4.UploadService(
-                user_access_token=self.user_items["user_upload_token"],
-                session_key=session_key,
-            )
+            upload_service = upload_api_v4.UploadService(user_session, session_key)
 
         return upload_service
 
     def _handle_upload_exception(
-        self, ex: Exception, progress: UploaderProgress
+        self, ex: BaseException, progress: UploaderProgress
     ) -> None:
-        retries = progress["retries"]
+        retries = progress.get("retries", 0)
         begin_offset = progress.get("begin_offset")
-        chunk_size = progress["chunk_size"]
+        offset = progress.get("offset")
 
         if retries <= constants.MAX_UPLOAD_RETRIES and _is_retriable_exception(ex):
-            self.emitter.emit("upload_interrupted", progress)
+            self.emitter.emit("upload_retrying", progress)
+
             LOG.warning(
-                # use %s instead of %d because offset could be None
-                "Error uploading chunk_size %d at begin_offset %s: %s: %s",
-                chunk_size,
-                begin_offset,
-                ex.__class__.__name__,
-                str(ex),
+                f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
             )
+
             # Keep things immutable here. Will increment retries in the caller
             retries += 1
-            if _is_immediate_retry(ex):
+            if _is_immediate_retriable_exception(ex):
                 sleep_for = 0
             else:
                 sleep_for = min(2**retries, 16)
             LOG.info(
-                "Retrying in %d seconds (%d/%d)",
-                sleep_for,
-                retries,
-                constants.MAX_UPLOAD_RETRIES,
+                f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
             )
             if sleep_for:
                 time.sleep(sleep_for)
         else:
+            self.emitter.emit("upload_failed", progress)
             raise ex
 
+    @classmethod
+    def _upload_name(cls, progress: UploaderProgress):
+        # Strictly speaking these sequence properties should not be exposed in this context
+        # TODO: Maybe move these logging statements to event handlers
+        sequence_uuid: str | None = T.cast(
+            T.Union[str, None], progress.get("sequence_uuid")
+        )
+        import_path = T.cast(T.Union[str, None], progress.get("import_path"))
+        if sequence_uuid is not None:
+            if import_path is None:
+                name: str = f"sequence_{sequence_uuid}"
+            else:
+                name = f"sequence_{sequence_uuid}/{Path(import_path).name}"
+        else:
+            name = Path(import_path or "unknown").name
+        return name
+
     def _chunk_with_progress_emitted(
-        self,
-        stream: T.IO[bytes],
-        progress: UploaderProgress,
+        self, stream: T.IO[bytes], progress: UploaderProgress
     ) -> T.Generator[bytes, None, None]:
         for chunk in upload_api_v4.UploadService.chunkize_byte_stream(
-            stream, self.chunk_size
+            stream, self.upload_options.chunk_size
         ):
             yield chunk
 
@@ -649,11 +1035,21 @@ class Uploader:
 
     def _upload_stream_retryable(
         self,
-        upload_service: upload_api_v4.UploadService,
+        user_session: requests.Session,
         fp: T.IO[bytes],
-        progress: UploaderProgress,
+        session_key: str,
+        progress: UploaderProgress | None = None,
     ) -> str:
         """Upload the stream with safe retries guraranteed"""
+        if progress is None:
+            progress = T.cast(UploaderProgress, {})
+
+        upload_service = self._create_upload_service(user_session, session_key)
+
+        if "entity_size" not in progress:
+            fp.seek(0, io.SEEK_END)
+            entity_size = fp.tell()
+            progress["entity_size"] = entity_size
 
         begin_offset = upload_service.fetch_offset()
 
@@ -662,11 +1058,39 @@ class Uploader:
 
         self.emitter.emit("upload_fetch_offset", progress)
 
-        fp.seek(begin_offset, io.SEEK_SET)
+        # Estimate the read timeout
+        if not constants.MIN_UPLOAD_SPEED:
+            read_timeout = None
+        else:
+            remaining_bytes = abs(progress["entity_size"] - begin_offset)
+            read_timeout = max(
+                api_v4.REQUESTS_TIMEOUT,
+                remaining_bytes / constants.MIN_UPLOAD_SPEED,
+            )
 
+        # Upload from begin_offset
+        fp.seek(begin_offset, io.SEEK_SET)
         shifted_chunks = self._chunk_with_progress_emitted(fp, progress)
 
-        return upload_service.upload_shifted_chunks(shifted_chunks, begin_offset)
+        # Start uploading
+        return upload_service.upload_shifted_chunks(
+            shifted_chunks, begin_offset, read_timeout=read_timeout
+        )
+
+    def _gen_session_key(self, fp: T.IO[bytes], progress: dict[str, T.Any]) -> str:
+        if self.upload_options.noresume:
+            # Generate a unique UUID for session_key when noresume is True
+            # to prevent resuming from previous uploads
+            session_key = f"{_prefixed_uuid4()}"
+        else:
+            fp.seek(0, io.SEEK_SET)
+            session_key = utils.md5sum_fp(fp).hexdigest()
+
+        filetype = progress.get("file_type")
+        if filetype is not None:
+            session_key = _suffix_session_key(session_key, types.FileType(filetype))
+
+        return session_key
 
 
 def _validate_metadatas(metadatas: T.Sequence[types.ImageMetadata]):
@@ -676,7 +1100,7 @@ def _validate_metadatas(metadatas: T.Sequence[types.ImageMetadata]):
             raise FileNotFoundError(f"No such file {metadata.filename}")
 
 
-def _is_immediate_retry(ex: Exception):
+def _is_immediate_retriable_exception(ex: BaseException) -> bool:
     if (
         isinstance(ex, requests.HTTPError)
         and isinstance(ex.response, requests.Response)
@@ -689,8 +1113,10 @@ def _is_immediate_retry(ex: Exception):
         # resp: {"debug_info":{"retriable":true,"type":"OffsetInvalidError","message":"Request starting offset is invalid"}}
         return resp.get("debug_info", {}).get("retriable", False)
 
+    return False
+
 
-def _is_retriable_exception(ex: Exception):
+def _is_retriable_exception(ex: BaseException) -> bool:
     if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
         return True
 
@@ -709,19 +1135,36 @@ def _is_retriable_exception(ex: Exception):
     return False
 
 
-def _session_key(
-    upload_md5sum: str, filetype: api_v4.ClusterFileType | types.FileType
+_SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
+    api_v4.ClusterFileType.ZIP: ".zip",
+    api_v4.ClusterFileType.CAMM: ".mp4",
+    api_v4.ClusterFileType.BLACKVUE: ".mp4",
+    types.FileType.IMAGE: ".jpg",
+    types.FileType.ZIP: ".zip",
+    types.FileType.BLACKVUE: ".mp4",
+    types.FileType.CAMM: ".mp4",
+    types.FileType.GOPRO: ".mp4",
+    types.FileType.VIDEO: ".mp4",
+}
+
+
+def _suffix_session_key(
+    key: str, filetype: api_v4.ClusterFileType | types.FileType
 ) -> str:
-    _SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
-        api_v4.ClusterFileType.ZIP: ".zip",
-        api_v4.ClusterFileType.CAMM: ".mp4",
-        api_v4.ClusterFileType.BLACKVUE: ".mp4",
-        types.FileType.IMAGE: ".jpg",
-        types.FileType.ZIP: ".zip",
-        types.FileType.BLACKVUE: ".mp4",
-        types.FileType.CAMM: ".mp4",
-        types.FileType.GOPRO: ".mp4",
-        types.FileType.VIDEO: ".mp4",
-    }
-
-    return f"mly_tools_{upload_md5sum}{_SUFFIX_MAP[filetype]}"
+    is_uuid_before = _is_uuid(key)
+
+    key = f"mly_tools_{key}{_SUFFIX_MAP[filetype]}"
+
+    assert _is_uuid(key) is is_uuid_before
+
+    return key
+
+
+def _prefixed_uuid4():
+    prefixed = f"uuid_{uuid.uuid4().hex}"
+    assert _is_uuid(prefixed)
+    return prefixed
+
+
+def _is_uuid(key: str) -> bool:
+    return key.startswith("uuid_") or key.startswith("mly_tools_uuid_")