mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0a1__py3-none-any.whl

mapillary_tools/upload.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import logging
 import os
@@ -7,15 +9,15 @@ import typing as T
 import uuid
 from pathlib import Path
 
+import jsonschema
 import requests
 from tqdm import tqdm
 
 from . import (
     api_v4,
-    authenticate,
-    config,
     constants,
     exceptions,
+    geo,
     history,
     ipc,
     telemetry,
@@ -25,31 +27,23 @@ from . import (
     utils,
     VERSION,
 )
-from .camm import camm_builder
-from .geotag import gpmf_parser
+from .camm import camm_builder, camm_parser
+from .gpmf import gpmf_parser
 from .mp4 import simple_mp4_builder
 from .types import FileType
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
 
 LOG = logging.getLogger(__name__)
-MAPILLARY_DISABLE_API_LOGGING = os.getenv("MAPILLARY_DISABLE_API_LOGGING")
-MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN = os.getenv(
-    "MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"
-)
-MAPILLARY__EXPERIMENTAL_ENABLE_IMU = os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU")
-CAMM_CONVERTABLES = {FileType.CAMM, FileType.BLACKVUE, FileType.GOPRO}
 
 
-class UploadError(Exception):
-    def __init__(self, inner_ex) -> None:
-        self.inner_ex = inner_ex
-        super().__init__(str(inner_ex))
+class UploadedAlreadyError(uploader.SequenceError):
+    pass
 
 
 def _load_validate_metadatas_from_desc_path(
-    desc_path: T.Optional[str], import_paths: T.Sequence[Path]
-) -> T.List[types.Metadata]:
+    desc_path: str | None, import_paths: T.Sequence[Path]
+) -> list[types.Metadata]:
     is_default_desc_path = False
     if desc_path is None:
         is_default_desc_path = True
@@ -67,7 +61,7 @@ def _load_validate_metadatas_from_desc_path(
                     "The description path must be specified (with --desc_path) when uploading a single file",
                 )
 
-    descs: T.List[types.DescriptionOrError] = []
+    descs: list[types.DescriptionOrError] = []
 
     if desc_path == "-":
         try:
@@ -75,7 +69,7 @@ def _load_validate_metadatas_from_desc_path(
         except json.JSONDecodeError as ex:
             raise exceptions.MapillaryInvalidDescriptionFile(
                 f"Invalid JSON stream from stdin: {ex}"
-            )
+            ) from ex
     else:
         if not os.path.isfile(desc_path):
             if is_default_desc_path:
@@ -92,7 +86,7 @@ def _load_validate_metadatas_from_desc_path(
             except json.JSONDecodeError as ex:
                 raise exceptions.MapillaryInvalidDescriptionFile(
                     f"Invalid JSON file {desc_path}: {ex}"
-                )
+                ) from ex
 
     # the descs load from stdin or json file may contain invalid entries
     validated_descs = [
@@ -120,7 +114,7 @@ def _load_validate_metadatas_from_desc_path(
 def zip_images(
     import_path: Path,
     zip_dir: Path,
-    desc_path: T.Optional[str] = None,
+    desc_path: str | None = None,
 ):
     if not import_path.is_dir():
         raise exceptions.MapillaryFileNotFoundError(
@@ -137,43 +131,19 @@ def zip_images(
         metadata for metadata in metadatas if isinstance(metadata, types.ImageMetadata)
     ]
 
-    uploader.zip_images(image_metadatas, zip_dir)
+    uploader.ZipImageSequence.zip_images(image_metadatas, zip_dir)
 
 
-def fetch_user_items(
-    user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None
-) -> types.UserItem:
-    if user_name is None:
-        all_user_items = config.list_all_users()
-        if not all_user_items:
-            raise exceptions.MapillaryBadParameterError(
-                "No Mapillary account found. Add one with --user_name"
-            )
-        if len(all_user_items) == 1:
-            user_items = all_user_items[0]
-        else:
-            raise exceptions.MapillaryBadParameterError(
-                "Found multiple Mapillary accounts. Please specify one with --user_name"
-            )
-    else:
-        user_items = authenticate.authenticate_user(user_name)
-
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
-        )
-        org = resp.json()
-        LOG.info("Uploading to organization: %s", json.dumps(org))
-        user_items = T.cast(
-            types.UserItem, {**user_items, "MAPOrganizationKey": organization_key}
-        )
-    return user_items
-
-
-def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None:
+def _setup_history(
+    emitter: uploader.EventEmitter,
+    upload_run_params: JSONDict,
+    metadatas: list[types.Metadata],
+) -> None:
     @emitter.on("upload_start")
-    def upload_start(payload: uploader.Progress):
-        md5sum = payload["md5sum"]
+    def check_duplication(payload: uploader.Progress):
+        md5sum = payload.get("md5sum")
+        assert md5sum is not None, f"md5sum has to be set for {payload}"
+
         if history.is_uploaded(md5sum):
             sequence_uuid = payload.get("sequence_uuid")
             if sequence_uuid is None:
@@ -189,19 +159,15 @@ def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None:
                     sequence_uuid,
                     history.history_desc_path(md5sum),
                 )
-            raise uploader.UploadCancelled()
+            raise UploadedAlreadyError()
 
-
-def _setup_write_upload_history(
-    emitter: uploader.EventEmitter,
-    params: JSONDict,
-    metadatas: T.Optional[T.List[types.Metadata]] = None,
-) -> None:
     @emitter.on("upload_finished")
-    def upload_finished(payload: uploader.Progress):
+    def write_history(payload: uploader.Progress):
         sequence_uuid = payload.get("sequence_uuid")
-        md5sum = payload["md5sum"]
-        if sequence_uuid is None or metadatas is None:
+        md5sum = payload.get("md5sum")
+        assert md5sum is not None, f"md5sum has to be set for {payload}"
+
+        if sequence_uuid is None:
             sequence = None
         else:
             sequence = [
@@ -211,10 +177,11 @@ def _setup_write_upload_history(
                 and metadata.MAPSequenceUUID == sequence_uuid
             ]
             sequence.sort(key=lambda metadata: metadata.sort_key())
+
         try:
             history.write_history(
                 md5sum,
-                params,
+                upload_run_params,
                 T.cast(JSONDict, payload),
                 sequence,
             )
@@ -223,7 +190,7 @@ def _setup_write_upload_history(
 
 
 def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
-    upload_pbar: T.Optional[tqdm] = None
+    upload_pbar: tqdm | None = None
 
     @emitter.on("upload_fetch_offset")
     def upload_fetch_offset(payload: uploader.Progress) -> None:
@@ -234,7 +201,7 @@ def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
 
         nth = payload["sequence_idx"] + 1
         total = payload["total_sequence_count"]
-        import_path: T.Optional[str] = payload.get("import_path")
+        import_path: str | None = payload.get("import_path")
         filetype = payload.get("file_type", "unknown").upper()
         if import_path is None:
             _desc = f"Uploading {filetype} ({nth}/{total})"
@@ -269,25 +236,40 @@ def _setup_ipc(emitter: uploader.EventEmitter):
     @emitter.on("upload_start")
     def upload_start(payload: uploader.Progress):
         type: uploader.EventName = "upload_start"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug("IPC %s: %s", type.upper(), payload)
         ipc.send(type, payload)
 
     @emitter.on("upload_fetch_offset")
     def upload_fetch_offset(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_fetch_offset"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug("IPC %s: %s", type.upper(), payload)
         ipc.send(type, payload)
 
     @emitter.on("upload_progress")
     def upload_progress(payload: uploader.Progress):
         type: uploader.EventName = "upload_progress"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+
+        if LOG.getEffectiveLevel() <= logging.DEBUG:
+            # In debug mode, we want to see the progress every 30 seconds
+            # instead of every chunk (which is too verbose)
+            INTERVAL_SECONDS = 30
+            now = time.time()
+            last_upload_progress_debug_at: float | None = T.cast(T.Dict, payload).get(
+                "_last_upload_progress_debug_at"
+            )
+            if (
+                last_upload_progress_debug_at is None
+                or last_upload_progress_debug_at + INTERVAL_SECONDS < now
+            ):
+                LOG.debug("IPC %s: %s", type.upper(), payload)
+                T.cast(T.Dict, payload)["_last_upload_progress_debug_at"] = now
+
         ipc.send(type, payload)
 
     @emitter.on("upload_end")
     def upload_end(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_end"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug("IPC %s: %s", type.upper(), payload)
         ipc.send(type, payload)
 
 
@@ -309,7 +291,7 @@ class _APIStats(uploader.Progress, total=False):
 
 
 def _setup_api_stats(emitter: uploader.EventEmitter):
-    all_stats: T.List[_APIStats] = []
+    all_stats: list[_APIStats] = []
 
     @emitter.on("upload_start")
     def collect_start_time(payload: _APIStats) -> None:
@@ -337,15 +319,18 @@ def _setup_api_stats(emitter: uploader.EventEmitter):
         now = time.time()
         payload["upload_end_time"] = now
         payload["upload_total_time"] += now - payload["upload_last_restart_time"]
+
+    @emitter.on("upload_finished")
+    def append_stats(payload: _APIStats) -> None:
         all_stats.append(payload)
 
     return all_stats
 
 
-def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
+def _summarize(stats: T.Sequence[_APIStats]) -> dict:
     total_image_count = sum(s.get("sequence_image_count", 0) for s in stats)
     total_uploaded_sequence_count = len(stats)
-    # note that stats[0]["total_sequence_count"] not always same as total_uploaded_sequence_count
+    # Note that stats[0]["total_sequence_count"] not always same as total_uploaded_sequence_count
 
     total_uploaded_size = sum(
         s["entity_size"] - s.get("upload_first_offset", 0) for s in stats
@@ -363,6 +348,7 @@ def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
 
     upload_summary = {
         "images": total_image_count,
+        # TODO: rename sequences to total uploads
         "sequences": total_uploaded_sequence_count,
         "size": round(total_entity_size_mb, 4),
         "uploaded_size": round(total_uploaded_size_mb, 4),
@@ -373,37 +359,34 @@ def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
     return upload_summary
 
 
-def _show_upload_summary(stats: T.Sequence[_APIStats]):
-    grouped: T.Dict[str, T.List[_APIStats]] = {}
-    for stat in stats:
-        grouped.setdefault(stat.get("file_type", "unknown"), []).append(stat)
+def _show_upload_summary(stats: T.Sequence[_APIStats], errors: T.Sequence[Exception]):
+    if not stats:
+        LOG.info("Nothing uploaded. Bye.")
+    else:
+        grouped: dict[str, list[_APIStats]] = {}
+        for stat in stats:
+            grouped.setdefault(stat.get("file_type", "unknown"), []).append(stat)
+
+        for file_type, typed_stats in grouped.items():
+            if file_type == FileType.IMAGE.value:
+                LOG.info("%8d image sequences uploaded", len(typed_stats))
+            else:
+                LOG.info("%8d  %s videos uploaded", len(typed_stats), file_type.upper())
 
-    for file_type, typed_stats in grouped.items():
-        if file_type == FileType.IMAGE.value:
-            LOG.info(
-                "%8d  %s sequences uploaded",
-                len(typed_stats),
-                file_type.upper(),
-            )
-        else:
-            LOG.info(
-                "%8d  %s files uploaded",
-                len(typed_stats),
-                file_type.upper(),
-            )
+        summary = _summarize(stats)
+        LOG.info("%8.1fM data in total", summary["size"])
+        LOG.info("%8.1fM data uploaded", summary["uploaded_size"])
+        LOG.info("%8.1fs upload time", summary["time"])
 
-    summary = _summarize(stats)
-    LOG.info("%8.1fM data in total", summary["size"])
-    LOG.info("%8.1fM data uploaded", summary["uploaded_size"])
-    LOG.info("%8.1fs upload time", summary["time"])
+    for error in errors:
+        LOG.error("Upload error: %s: %s", error.__class__.__name__, error)
 
 
-def _api_logging_finished(summary: T.Dict):
-    if MAPILLARY_DISABLE_API_LOGGING:
+def _api_logging_finished(summary: dict):
+    if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     action: api_v4.ActionType = "upload_finished_upload"
-    LOG.debug("API Logging for action %s: %s", action, summary)
     try:
         api_v4.log_event(action, summary)
     except requests.HTTPError as exc:
@@ -416,13 +399,12 @@ def _api_logging_finished(summary: T.Dict):
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
 
 
-def _api_logging_failed(payload: T.Dict, exc: Exception):
-    if MAPILLARY_DISABLE_API_LOGGING:
+def _api_logging_failed(payload: dict, exc: Exception):
+    if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     payload_with_reason = {**payload, "reason": exc.__class__.__name__}
     action: api_v4.ActionType = "upload_failed_upload"
-    LOG.debug("API Logging for action %s: %s", action, payload)
     try:
         api_v4.log_event(action, payload_with_reason)
     except requests.HTTPError as exc:
@@ -436,18 +418,14 @@ def _api_logging_failed(payload: T.Dict, exc: Exception):
 
 
 def _load_descs(
-    _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]],
-    desc_path: T.Optional[str],
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
+    desc_path: str | None,
     import_paths: T.Sequence[Path],
-) -> T.List[types.Metadata]:
-    metadatas: T.List[types.Metadata]
+) -> list[types.Metadata]:
+    metadatas: list[types.Metadata]
 
     if _metadatas_from_process is not None:
-        metadatas = [
-            metadata
-            for metadata in _metadatas_from_process
-            if not isinstance(metadata, types.ErrorMetadata)
-        ]
+        metadatas, _ = types.separate_errors(_metadatas_from_process)
     else:
         metadatas = _load_validate_metadatas_from_desc_path(desc_path, import_paths)
 
@@ -471,31 +449,139 @@ _M = T.TypeVar("_M", bound=types.Metadata)
 
 
 def _find_metadata_with_filename_existed_in(
-    metadatas: T.Sequence[_M], paths: T.Sequence[Path]
-) -> T.List[_M]:
+    metadatas: T.Iterable[_M], paths: T.Iterable[Path]
+) -> list[_M]:
     resolved_image_paths = set(p.resolve() for p in paths)
     return [d for d in metadatas if d.filename.resolve() in resolved_image_paths]
 
 
-def upload(
-    import_path: T.Union[Path, T.Sequence[Path]],
-    desc_path: T.Optional[str] = None,
-    _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None,
-    user_name: T.Optional[str] = None,
-    organization_key: T.Optional[str] = None,
-    dry_run=False,
-    skip_subfolders=False,
-) -> None:
-    import_paths: T.Sequence[Path]
+def _gen_upload_everything(
+    mly_uploader: uploader.Uploader,
+    metadatas: T.Sequence[types.Metadata],
+    import_paths: T.Sequence[Path],
+    skip_subfolders: bool,
+):
+    # Upload images
+    image_metadatas = _find_metadata_with_filename_existed_in(
+        (m for m in metadatas if isinstance(m, types.ImageMetadata)),
+        utils.find_images(import_paths, skip_subfolders=skip_subfolders),
+    )
+    for image_result in uploader.ZipImageSequence.prepare_images_and_upload(
+        image_metadatas,
+        mly_uploader,
+    ):
+        yield image_result
+
+    # Upload videos
+    video_metadatas = _find_metadata_with_filename_existed_in(
+        (m for m in metadatas if isinstance(m, types.VideoMetadata)),
+        utils.find_videos(import_paths, skip_subfolders=skip_subfolders),
+    )
+    for video_result in _gen_upload_videos(mly_uploader, video_metadatas):
+        yield video_result
+
+    # Upload zip files
+    zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
+    for zip_result in _gen_upload_zipfiles(mly_uploader, zip_paths):
+        yield zip_result
+
+
+def _gen_upload_videos(
+    mly_uploader: uploader.Uploader, video_metadatas: T.Sequence[types.VideoMetadata]
+) -> T.Generator[tuple[types.VideoMetadata, uploader.UploadResult], None, None]:
+    for idx, video_metadata in enumerate(video_metadatas):
+        try:
+            video_metadata.update_md5sum()
+        except Exception as ex:
+            yield video_metadata, uploader.UploadResult(error=ex)
+            continue
+
+        assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
+
+        # Convert video metadata to CAMMInfo
+        camm_info = _prepare_camm_info(video_metadata)
+
+        # Create the CAMM sample generator
+        camm_sample_generator = camm_builder.camm_sample_generator2(camm_info)
+
+        progress: uploader.SequenceProgress = {
+            "total_sequence_count": len(video_metadatas),
+            "sequence_idx": idx,
+            "file_type": video_metadata.filetype.value,
+            "import_path": str(video_metadata.filename),
+            "md5sum": video_metadata.md5sum,
+        }
+
+        session_key = uploader._session_key(
+            video_metadata.md5sum, upload_api_v4.ClusterFileType.CAMM
+        )
+
+        try:
+            with video_metadata.filename.open("rb") as src_fp:
+                # Build the mp4 stream with the CAMM samples
+                camm_fp = simple_mp4_builder.transform_mp4(
+                    src_fp, camm_sample_generator
+                )
+
+                # Upload the mp4 stream
+                cluster_id = mly_uploader.upload_stream(
+                    T.cast(T.IO[bytes], camm_fp),
+                    upload_api_v4.ClusterFileType.CAMM,
+                    session_key,
+                    progress=T.cast(T.Dict[str, T.Any], progress),
+                )
+        except Exception as ex:
+            yield video_metadata, uploader.UploadResult(error=ex)
+        else:
+            yield video_metadata, uploader.UploadResult(result=cluster_id)
+
+
+def _prepare_camm_info(video_metadata: types.VideoMetadata) -> camm_parser.CAMMInfo:
+    camm_info = camm_parser.CAMMInfo(
+        make=video_metadata.make or "", model=video_metadata.model or ""
+    )
+
+    for point in video_metadata.points:
+        if isinstance(point, telemetry.CAMMGPSPoint):
+            if camm_info.gps is None:
+                camm_info.gps = []
+            camm_info.gps.append(point)
+
+        elif isinstance(point, telemetry.GPSPoint):
+            # There is no proper CAMM entry for GoPro GPS
+            if camm_info.mini_gps is None:
+                camm_info.mini_gps = []
+            camm_info.mini_gps.append(point)
+
+        elif isinstance(point, geo.Point):
+            if camm_info.mini_gps is None:
+                camm_info.mini_gps = []
+            camm_info.mini_gps.append(point)
+        else:
+            raise ValueError(f"Unknown point type: {point}")
+
+    if constants.MAPILLARY__EXPERIMENTAL_ENABLE_IMU:
+        if video_metadata.filetype is FileType.GOPRO:
+            with video_metadata.filename.open("rb") as fp:
+                gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True)
+            if gopro_info is not None:
+                camm_info.accl = gopro_info.accl or []
+                camm_info.gyro = gopro_info.gyro or []
+                camm_info.magn = gopro_info.magn or []
+
+    return camm_info
+
+
+def _normalize_import_paths(import_path: Path | T.Sequence[Path]) -> list[Path]:
+    import_paths: list[Path]
+
     if isinstance(import_path, Path):
         import_paths = [import_path]
     else:
         assert isinstance(import_path, list)
         import_paths = import_path
-    import_paths = list(utils.deduplicate_paths(import_paths))
 
-    if not import_paths:
-        return
+    import_paths = list(utils.deduplicate_paths(import_paths))
 
     # Check and fail early
     for path in import_paths:
@@ -504,176 +590,140 @@ def upload(
                 f"Import file or directory not found: {path}"
             )
 
+    return import_paths
+
+
+def _continue_or_fail(ex: Exception) -> Exception:
+    """
+    Wrap the exception, or re-raise if it is a fatal error (i.e. there is no point to continue)
+    """
+
+    if isinstance(ex, uploader.SequenceError):
+        return ex
+
+    # Certain files not found or no permission
+    if isinstance(ex, OSError):
+        return ex
+
+    # Certain metadatas are not valid
+    if isinstance(ex, exceptions.MapillaryMetadataValidationError):
+        return ex
+
+    # Fatal error: this is thrown after all retries
+    if isinstance(ex, requests.ConnectionError):
+        raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
+
+    # Fatal error: this is thrown after all retries
+    if isinstance(ex, requests.Timeout):
+        raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
+
+    # Fatal error:
+    if isinstance(ex, requests.HTTPError) and isinstance(
+        ex.response, requests.Response
+    ):
+        if api_v4.is_auth_error(ex.response):
+            raise exceptions.MapillaryUploadUnauthorizedError(
+                api_v4.extract_auth_error_message(ex.response)
+            ) from ex
+        raise ex
+
+    raise ex
+
+
+def upload(
+    import_path: Path | T.Sequence[Path],
+    user_items: types.UserItem,
+    desc_path: str | None = None,
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
+    dry_run=False,
+    skip_subfolders=False,
+) -> None:
+    import_paths = _normalize_import_paths(import_path)
+
     metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
 
-    user_items = fetch_user_items(user_name, organization_key)
+    jsonschema.validate(instance=user_items, schema=types.UserItemSchema)
 
     # Setup the emitter -- the order matters here
 
     emitter = uploader.EventEmitter()
 
-    enable_history = history.MAPILLARY_UPLOAD_HISTORY_PATH and (
-        not dry_run or MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN == "YES"
+    # When dry_run mode is on, we disable history by default.
+    # But we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
+    # and when it is on, we enable history regardless of dry_run
+    enable_history = constants.MAPILLARY_UPLOAD_HISTORY_PATH and (
+        not dry_run or constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
     )
 
-    # Put it first one to cancel early
+    # Put it first one to check duplications first
     if enable_history:
-        _setup_cancel_due_to_duplication(emitter)
+        upload_run_params: JSONDict = {
+            # Null if multiple paths provided
+            "import_path": str(import_path) if isinstance(import_path, Path) else None,
+            "organization_key": user_items.get("MAPOrganizationKey"),
+            "user_key": user_items.get("MAPSettingsUserKey"),
+            "version": VERSION,
+            "run_at": time.time(),
+        }
+        _setup_history(emitter, upload_run_params, metadatas)
 
-    # This one set up tdqm
+    # Set up tdqm
     _setup_tdqm(emitter)
 
-    # Now stats is empty but it will collect during upload
+    # Now stats is empty but it will collect during ALL uploads
     stats = _setup_api_stats(emitter)
 
-    # Send the progress as well as the log stats collected above
+    # Send the progress via IPC, and log the progress in debug mode
     _setup_ipc(emitter)
 
-    params: JSONDict = {
-        # null if multiple paths provided
-        "import_path": str(import_path) if isinstance(import_path, Path) else None,
-        "organization_key": user_items.get("MAPOrganizationKey"),
-        "user_key": user_items.get("MAPSettingsUserKey"),
-        "version": VERSION,
-    }
-
-    if enable_history:
-        _setup_write_upload_history(emitter, params, metadatas)
+    mly_uploader = uploader.Uploader(user_items, emitter=emitter, dry_run=dry_run)
 
-    mly_uploader = uploader.Uploader(
-        user_items,
-        emitter=emitter,
-        dry_run=dry_run,
-        chunk_size=int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
+    results = _gen_upload_everything(
+        mly_uploader, metadatas, import_paths, skip_subfolders
     )
 
-    try:
-        image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
-        # find descs that match the image paths from the import paths
-        image_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.ImageMetadata)
-        ]
-        specified_image_metadatas = _find_metadata_with_filename_existed_in(
-            image_metadatas, image_paths
-        )
-        if specified_image_metadatas:
-            try:
-                clusters = mly_uploader.upload_images(
-                    specified_image_metadatas,
-                    event_payload={"file_type": FileType.IMAGE.value},
-                )
-            except Exception as ex:
-                raise UploadError(ex) from ex
-
-            if clusters:
-                LOG.debug("Uploaded to cluster: %s", clusters)
-
-        video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
-        video_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.VideoMetadata)
-        ]
-        specified_video_metadatas = _find_metadata_with_filename_existed_in(
-            video_metadatas, video_paths
-        )
-        for idx, video_metadata in enumerate(specified_video_metadatas):
-            video_metadata.update_md5sum()
-            assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
-
-            # extract telemetry measurements from GoPro videos
-            telemetry_measurements: T.List[telemetry.TelemetryMeasurement] = []
-            if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
-                if video_metadata.filetype is FileType.GOPRO:
-                    with video_metadata.filename.open("rb") as fp:
-                        telemetry_data = gpmf_parser.extract_telemetry_data(fp)
-                    if telemetry_data:
-                        telemetry_measurements.extend(telemetry_data.accl)
-                        telemetry_measurements.extend(telemetry_data.gyro)
-                        telemetry_measurements.extend(telemetry_data.magn)
-                    telemetry_measurements.sort(key=lambda m: m.time)
-
-            generator = camm_builder.camm_sample_generator2(
-                video_metadata, telemetry_measurements=telemetry_measurements
-            )
+    upload_successes = 0
+    upload_errors: list[Exception] = []
 
-            with video_metadata.filename.open("rb") as src_fp:
-                camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
-                event_payload: uploader.Progress = {
-                    "total_sequence_count": len(specified_video_metadatas),
-                    "sequence_idx": idx,
-                    "file_type": video_metadata.filetype.value,
-                    "import_path": str(video_metadata.filename),
-                }
-                try:
-                    cluster_id = mly_uploader.upload_stream(
-                        T.cast(T.BinaryIO, camm_fp),
-                        upload_api_v4.ClusterFileType.CAMM,
-                        video_metadata.md5sum,
-                        event_payload=event_payload,
-                    )
-                except Exception as ex:
-                    raise UploadError(ex) from ex
-                LOG.debug("Uploaded to cluster: %s", cluster_id)
-
-        zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
-        _upload_zipfiles(mly_uploader, zip_paths)
-
-    except UploadError as ex:
-        inner_ex = ex.inner_ex
+    # The real upload happens sequentially here
+    try:
+        for _, result in results:
+            if result.error is not None:
+                upload_errors.append(_continue_or_fail(result.error))
+            else:
+                upload_successes += 1
 
+    except Exception as ex:
+        # Fatal error: log and raise
         if not dry_run:
-            _api_logging_failed(_summarize(stats), inner_ex)
-
-        if isinstance(inner_ex, requests.ConnectionError):
-            raise exceptions.MapillaryUploadConnectionError(str(inner_ex)) from inner_ex
-
-        if isinstance(inner_ex, requests.Timeout):
-            raise exceptions.MapillaryUploadTimeoutError(str(inner_ex)) from inner_ex
-
-        if isinstance(inner_ex, requests.HTTPError) and isinstance(
-            inner_ex.response, requests.Response
-        ):
-            if inner_ex.response.status_code in [400, 401]:
-                try:
-                    error_body = inner_ex.response.json()
-                except Exception:
-                    error_body = {}
-                debug_info = error_body.get("debug_info", {})
-                if debug_info.get("type") in ["NotAuthorizedError"]:
-                    raise exceptions.MapillaryUploadUnauthorizedError(
-                        debug_info.get("message")
-                    ) from inner_ex
-            raise inner_ex
-
-        raise inner_ex
-
-    if stats:
+            _api_logging_failed(_summarize(stats), ex)
+        raise ex
+
+    else:
         if not dry_run:
             _api_logging_finished(_summarize(stats))
-        _show_upload_summary(stats)
-    else:
-        LOG.info("Nothing uploaded. Bye.")
+
+    finally:
+        # We collected stats after every upload is finished
+        assert upload_successes == len(stats)
+        _show_upload_summary(stats, upload_errors)
 
 
-def _upload_zipfiles(
+def _gen_upload_zipfiles(
     mly_uploader: uploader.Uploader,
     zip_paths: T.Sequence[Path],
-) -> None:
+) -> T.Generator[tuple[Path, uploader.UploadResult], None, None]:
     for idx, zip_path in enumerate(zip_paths):
-        event_payload: uploader.Progress = {
+        progress: uploader.SequenceProgress = {
             "total_sequence_count": len(zip_paths),
             "sequence_idx": idx,
-            "file_type": FileType.ZIP.value,
             "import_path": str(zip_path),
         }
         try:
-            cluster_id = mly_uploader.upload_zipfile(
-                zip_path, event_payload=event_payload
+            cluster_id = uploader.ZipImageSequence.prepare_zipfile_and_upload(
+                zip_path, mly_uploader, progress=T.cast(T.Dict[str, T.Any], progress)
             )
         except Exception as ex:
-            raise UploadError(ex) from ex
-
-        LOG.debug("Uploaded to cluster: %s", cluster_id)
+            yield zip_path, uploader.UploadResult(error=ex)
+        else:
+            yield zip_path, uploader.UploadResult(result=cluster_id)