mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/upload.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import logging
 import os
@@ -7,127 +9,134 @@ import typing as T
 import uuid
 from pathlib import Path
 
+import humanize
+import jsonschema
 import requests
 from tqdm import tqdm
 
 from . import (
     api_v4,
-    authenticate,
     config,
     constants,
     exceptions,
     history,
     ipc,
-    telemetry,
     types,
-    upload_api_v4,
     uploader,
     utils,
     VERSION,
 )
-from .camm import camm_builder
-from .geotag import gpmf_parser
-from .mp4 import simple_mp4_builder
+from .serializer.description import DescriptionJSONSerializer
 from .types import FileType
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
 
 LOG = logging.getLogger(__name__)
-MAPILLARY_DISABLE_API_LOGGING = os.getenv("MAPILLARY_DISABLE_API_LOGGING")
-MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN = os.getenv(
-    "MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"
-)
-MAPILLARY__EXPERIMENTAL_ENABLE_IMU = os.getenv("MAPILLARY__EXPERIMENTAL_ENABLE_IMU")
-CAMM_CONVERTABLES = {FileType.CAMM, FileType.BLACKVUE, FileType.GOPRO}
 
 
-class UploadError(Exception):
-    def __init__(self, inner_ex) -> None:
-        self.inner_ex = inner_ex
-        super().__init__(str(inner_ex))
+class UploadedAlready(uploader.SequenceError):
+    pass
 
 
-def _load_validate_metadatas_from_desc_path(
-    desc_path: T.Optional[str], import_paths: T.Sequence[Path]
-) -> T.List[types.Metadata]:
-    is_default_desc_path = False
-    if desc_path is None:
-        is_default_desc_path = True
-        if len(import_paths) == 1 and import_paths[0].is_dir():
-            desc_path = str(
-                import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME)
-            )
-        else:
-            if 1 < len(import_paths):
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading multiple paths",
-                )
-            else:
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading a single file",
-                )
+def upload(
+    import_path: Path | T.Sequence[Path],
+    user_items: config.UserItem,
+    desc_path: str | None = None,
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
+    reupload: bool = False,
+    dry_run: bool = False,
+    nofinish: bool = False,
+    noresume: bool = False,
+    skip_subfolders: bool = False,
+) -> None:
+    LOG.info("==> Uploading...")
 
-    descs: T.List[types.DescriptionOrError] = []
+    import_paths = _normalize_import_paths(import_path)
 
-    if desc_path == "-":
-        try:
-            descs = json.load(sys.stdin)
-        except json.JSONDecodeError as ex:
-            raise exceptions.MapillaryInvalidDescriptionFile(
-                f"Invalid JSON stream from stdin: {ex}"
-            )
-    else:
-        if not os.path.isfile(desc_path):
-            if is_default_desc_path:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found. Has the directory been processed yet?"
-                )
-            else:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found"
-                )
-        with open(desc_path) as fp:
-            try:
-                descs = json.load(fp)
-            except json.JSONDecodeError as ex:
-                raise exceptions.MapillaryInvalidDescriptionFile(
-                    f"Invalid JSON file {desc_path}: {ex}"
-                )
+    metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
 
-    # the descs load from stdin or json file may contain invalid entries
-    validated_descs = [
-        types.validate_and_fail_desc(desc)
-        for desc in descs
-        # skip error descriptions
-        if "error" not in desc
-    ]
+    jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
+
+    # Setup the emitter -- the order matters here
 
-    # throw if we found any invalid descs
-    invalid_descs = [desc for desc in validated_descs if "error" in desc]
-    if invalid_descs:
-        for desc in invalid_descs:
-            LOG.error("Invalid description entry: %s", json.dumps(desc))
-        raise exceptions.MapillaryInvalidDescriptionFile(
-            f"Found {len(invalid_descs)} invalid descriptions"
+    emitter = uploader.EventEmitter()
+
+    # Check duplications first
+    if not _is_history_disabled(dry_run):
+        upload_run_params: JSONDict = {
+            # Null if multiple paths provided
+            "import_path": str(import_path) if isinstance(import_path, Path) else None,
+            "organization_key": user_items.get("MAPOrganizationKey"),
+            "user_key": user_items.get("MAPSettingsUserKey"),
+            "version": VERSION,
+            "run_at": time.time(),
+        }
+        _setup_history(
+            emitter, upload_run_params, metadatas, reupload=reupload, nofinish=nofinish
         )
 
-    # validated_descs should contain no errors
-    return [
-        types.from_desc(T.cast(types.Description, desc)) for desc in validated_descs
-    ]
+    # Set up tdqm
+    _setup_tdqm(emitter)
 
+    # Now stats is empty but it will collect during ALL uploads
+    stats = _setup_api_stats(emitter)
 
-def zip_images(
-    import_path: Path,
-    zip_dir: Path,
-    desc_path: T.Optional[str] = None,
-):
+    # Send the progress via IPC, and log the progress in debug mode
+    _setup_ipc(emitter)
+
+    mly_uploader = uploader.Uploader(
+        uploader.UploadOptions(
+            user_items,
+            dry_run=dry_run,
+            nofinish=nofinish,
+            noresume=noresume,
+        ),
+        emitter=emitter,
+    )
+
+    results = _gen_upload_everything(
+        mly_uploader, metadatas, import_paths, skip_subfolders
+    )
+
+    upload_successes = 0
+    upload_errors: list[Exception] = []
+
+    # The real uploading happens sequentially here
+    try:
+        for _, result in results:
+            if result.error is not None:
+                upload_error = _continue_or_fail(result.error)
+                log_exception(upload_error)
+                upload_errors.append(upload_error)
+            else:
+                upload_successes += 1
+
+    except Exception as ex:
+        # Fatal error: log and raise
+        _api_logging_failed(_summarize(stats), ex, dry_run=dry_run)
+        raise ex
+
+    except KeyboardInterrupt:
+        LOG.info("Upload interrupted by user...")
+
+    else:
+        _api_logging_finished(_summarize(stats), dry_run=dry_run)
+
+    finally:
+        # We collected stats after every upload is finished
+        assert upload_successes == len(stats), (
+            f"Expect {upload_successes} success but got {stats}"
+        )
+        _show_upload_summary(stats, upload_errors)
+
+
+def zip_images(import_path: Path, zip_dir: Path, desc_path: str | None = None):
     if not import_path.is_dir():
         raise exceptions.MapillaryFileNotFoundError(
             f"Import directory not found: {import_path}"
         )
 
-    metadatas = _load_validate_metadatas_from_desc_path(desc_path, [import_path])
+    metadatas = _load_valid_metadatas_from_desc_path([import_path], desc_path)
 
     if not metadatas:
         LOG.warning("No images or videos found in %s", desc_path)
@@ -137,71 +146,99 @@ def zip_images(
         metadata for metadata in metadatas if isinstance(metadata, types.ImageMetadata)
     ]
 
-    uploader.zip_images(image_metadatas, zip_dir)
+    uploader.ZipUploader.zip_images(image_metadatas, zip_dir)
 
 
-def fetch_user_items(
-    user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None
-) -> types.UserItem:
-    if user_name is None:
-        all_user_items = config.list_all_users()
-        if not all_user_items:
-            raise exceptions.MapillaryBadParameterError(
-                "No Mapillary account found. Add one with --user_name"
-            )
-        if len(all_user_items) == 1:
-            user_items = all_user_items[0]
-        else:
-            raise exceptions.MapillaryBadParameterError(
-                "Found multiple Mapillary accounts. Please specify one with --user_name"
-            )
+def log_exception(ex: Exception) -> None:
+    if LOG.getEffectiveLevel() <= logging.DEBUG:
+        exc_info = ex
     else:
-        user_items = authenticate.authenticate_user(user_name)
-
-    if organization_key is not None:
-        resp = api_v4.fetch_organization(
-            user_items["user_upload_token"], organization_key
+        exc_info = None
+
+    exc_name = ex.__class__.__name__
+
+    if isinstance(ex, UploadedAlready):
+        LOG.info(f"{exc_name}: {ex}")
+    elif isinstance(ex, requests.HTTPError):
+        LOG.error(f"{exc_name}: {api_v4.readable_http_error(ex)}", exc_info=exc_info)
+    elif isinstance(ex, api_v4.HTTPContentError):
+        LOG.error(
+            f"{exc_name}: {ex}: {api_v4.readable_http_response(ex.response)}",
+            exc_info=exc_info,
         )
-        org = resp.json()
-        LOG.info("Uploading to organization: %s", json.dumps(org))
-        user_items = T.cast(
-            types.UserItem, {**user_items, "MAPOrganizationKey": organization_key}
-        )
-    return user_items
+    else:
+        LOG.error(f"{exc_name}: {ex}", exc_info=exc_info)
+
 
+def _is_history_disabled(dry_run: bool) -> bool:
+    # There is no way to read/write history if the path is not set
+    if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
+        return True
 
-def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None:
+    if dry_run:
+        # When dry_run mode is on, we disable history by default
+        # However, we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
+        # and when it is on, we enable history regardless of dry_run
+        if constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN:
+            return False
+        else:
+            return True
+
+    return False
+
+
+def _setup_history(
+    emitter: uploader.EventEmitter,
+    upload_run_params: JSONDict,
+    metadatas: list[types.Metadata],
+    reupload: bool,
+    nofinish: bool,
+) -> None:
     @emitter.on("upload_start")
-    def upload_start(payload: uploader.Progress):
-        md5sum = payload["md5sum"]
-        if history.is_uploaded(md5sum):
+    def check_duplication(payload: uploader.Progress):
+        md5sum = payload.get("sequence_md5sum")
+        assert md5sum is not None, f"md5sum has to be set for {payload}"
+
+        record = history.read_history_record(md5sum)
+
+        if record is not None:
             sequence_uuid = payload.get("sequence_uuid")
+            history_desc_path = history.history_desc_path(md5sum)
+            uploaded_at = record.get("summary", {}).get("upload_end_time", None)
+
             if sequence_uuid is None:
                 basename = os.path.basename(payload.get("import_path", ""))
-                LOG.info(
-                    "File %s has been uploaded already. Check the upload history at %s",
-                    basename,
-                    history.history_desc_path(md5sum),
-                )
+                name = f"file {basename}"
+
             else:
-                LOG.info(
-                    "Sequence %s has been uploaded already. Check the upload history at %s",
-                    sequence_uuid,
-                    history.history_desc_path(md5sum),
-                )
-            raise uploader.UploadCancelled()
+                name = f"sequence {sequence_uuid}"
 
+            if reupload:
+                if uploaded_at is not None:
+                    LOG.info(
+                        f"Reuploading {name}: previously uploaded {humanize.naturaldelta(time.time() - uploaded_at)} ago ({time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(uploaded_at))})"
+                    )
+                else:
+                    LOG.info(
+                        f"Reuploading {name}: already uploaded, see {history_desc_path}"
+                    )
+            else:
+                if uploaded_at is not None:
+                    msg = f"Skipping {name}: previously uploaded {humanize.naturaldelta(time.time() - uploaded_at)} ago ({time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(uploaded_at))})"
+                else:
+                    msg = f"Skipping {name}: already uploaded, see {history_desc_path}"
+                raise UploadedAlready(msg)
 
-def _setup_write_upload_history(
-    emitter: uploader.EventEmitter,
-    params: JSONDict,
-    metadatas: T.Optional[T.List[types.Metadata]] = None,
-) -> None:
     @emitter.on("upload_finished")
-    def upload_finished(payload: uploader.Progress):
+    def write_history(payload: uploader.Progress):
+        if nofinish:
+            return
+
         sequence_uuid = payload.get("sequence_uuid")
-        md5sum = payload["md5sum"]
-        if sequence_uuid is None or metadatas is None:
+        md5sum = payload.get("sequence_md5sum")
+        assert md5sum is not None, f"md5sum has to be set for {payload}"
+
+        if sequence_uuid is None:
             sequence = None
         else:
             sequence = [
@@ -211,22 +248,20 @@ def _setup_write_upload_history(
                 and metadata.MAPSequenceUUID == sequence_uuid
             ]
             sequence.sort(key=lambda metadata: metadata.sort_key())
+
         try:
             history.write_history(
-                md5sum,
-                params,
-                T.cast(JSONDict, payload),
-                sequence,
+                md5sum, upload_run_params, T.cast(JSONDict, payload), sequence
             )
         except OSError:
             LOG.warning("Error writing upload history %s", md5sum, exc_info=True)
 
 
 def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
-    upload_pbar: T.Optional[tqdm] = None
+    upload_pbar: tqdm | None = None
 
-    @emitter.on("upload_fetch_offset")
-    def upload_fetch_offset(payload: uploader.Progress) -> None:
+    @emitter.on("upload_start")
+    def upload_start(payload: uploader.Progress) -> None:
         nonlocal upload_pbar
 
         if upload_pbar is not None:
@@ -234,30 +269,55 @@ def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
 
         nth = payload["sequence_idx"] + 1
         total = payload["total_sequence_count"]
-        import_path: T.Optional[str] = payload.get("import_path")
+        import_path: str | None = payload.get("import_path")
         filetype = payload.get("file_type", "unknown").upper()
         if import_path is None:
-            _desc = f"Uploading {filetype} ({nth}/{total})"
+            desc = f"Uploading {filetype} ({nth}/{total})"
         else:
-            _desc = (
+            desc = (
                 f"Uploading {filetype} {os.path.basename(import_path)} ({nth}/{total})"
             )
         upload_pbar = tqdm(
             total=payload["entity_size"],
-            desc=_desc,
+            desc=desc,
             unit="B",
             unit_scale=True,
             unit_divisor=1024,
-            initial=payload["offset"],
+            initial=payload.get("offset", 0),
             disable=LOG.getEffectiveLevel() <= logging.DEBUG,
         )
 
+    @emitter.on("upload_fetch_offset")
+    def upload_fetch_offset(payload: uploader.Progress) -> None:
+        assert upload_pbar is not None, (
+            "progress_bar must be initialized in upload_start"
+        )
+        begin_offset = payload.get("begin_offset", 0)
+        if begin_offset is not None and begin_offset > 0:
+            if upload_pbar.total is not None:
+                progress_percent = (begin_offset / upload_pbar.total) * 100
+                upload_pbar.write(
+                    f"Resuming upload at {begin_offset=} ({progress_percent:3.0f}%)",
+                    file=sys.stderr,
+                )
+            else:
+                upload_pbar.write(
+                    f"Resuming upload at {begin_offset=}", file=sys.stderr
+                )
+            upload_pbar.reset()
+            upload_pbar.update(begin_offset)
+            upload_pbar.refresh()
+
     @emitter.on("upload_progress")
     def upload_progress(payload: uploader.Progress) -> None:
-        assert upload_pbar is not None, "progress_bar must be initialized"
+        assert upload_pbar is not None, (
+            "progress_bar must be initialized in upload_start"
+        )
         upload_pbar.update(payload["chunk_size"])
+        upload_pbar.refresh()
 
     @emitter.on("upload_end")
+    @emitter.on("upload_failed")
     def upload_end(_: uploader.Progress) -> None:
         nonlocal upload_pbar
         if upload_pbar:
@@ -269,25 +329,46 @@ def _setup_ipc(emitter: uploader.EventEmitter):
     @emitter.on("upload_start")
     def upload_start(payload: uploader.Progress):
         type: uploader.EventName = "upload_start"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
     @emitter.on("upload_fetch_offset")
     def upload_fetch_offset(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_fetch_offset"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
     @emitter.on("upload_progress")
     def upload_progress(payload: uploader.Progress):
         type: uploader.EventName = "upload_progress"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+
+        if LOG.getEffectiveLevel() <= logging.DEBUG:
+            # In debug mode, we want to see the progress every 30 seconds
+            # instead of every chunk (which is too verbose)
+            INTERVAL_SECONDS = 30
+            now = time.time()
+            last_upload_progress_debug_at: float | None = T.cast(T.Dict, payload).get(
+                "_last_upload_progress_debug_at"
+            )
+            if (
+                last_upload_progress_debug_at is None
+                or last_upload_progress_debug_at + INTERVAL_SECONDS < now
+            ):
+                LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
+                T.cast(T.Dict, payload)["_last_upload_progress_debug_at"] = now
+
         ipc.send(type, payload)
 
     @emitter.on("upload_end")
     def upload_end(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_end"
-        LOG.debug("Sending %s via IPC: %s", type, payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
+        ipc.send(type, payload)
+
+    @emitter.on("upload_failed")
+    def upload_failed(payload: uploader.Progress) -> None:
+        type: uploader.EventName = "upload_failed"
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
 
@@ -309,12 +390,17 @@ class _APIStats(uploader.Progress, total=False):
 
 
 def _setup_api_stats(emitter: uploader.EventEmitter):
-    all_stats: T.List[_APIStats] = []
+    all_stats: list[_APIStats] = []
 
     @emitter.on("upload_start")
     def collect_start_time(payload: _APIStats) -> None:
-        payload["upload_start_time"] = time.time()
+        now = time.time()
+        payload["upload_start_time"] = now
         payload["upload_total_time"] = 0
+        # These filed should be initialized in upload events like "upload_fetch_offset"
+        # but since we disabled them for uploading images, so we initialize them here
+        payload["upload_last_restart_time"] = now
+        payload["upload_first_offset"] = 0
 
     @emitter.on("upload_fetch_offset")
     def collect_restart_time(payload: _APIStats) -> None:
@@ -337,15 +423,18 @@ def _setup_api_stats(emitter: uploader.EventEmitter):
         now = time.time()
         payload["upload_end_time"] = now
         payload["upload_total_time"] += now - payload["upload_last_restart_time"]
+
+    @emitter.on("upload_finished")
+    def append_stats(payload: _APIStats) -> None:
         all_stats.append(payload)
 
     return all_stats
 
 
-def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
+def _summarize(stats: T.Sequence[_APIStats]) -> dict:
     total_image_count = sum(s.get("sequence_image_count", 0) for s in stats)
     total_uploaded_sequence_count = len(stats)
-    # note that stats[0]["total_sequence_count"] not always same as total_uploaded_sequence_count
+    # Note that stats[0]["total_sequence_count"] not always same as total_uploaded_sequence_count
 
     total_uploaded_size = sum(
         s["entity_size"] - s.get("upload_first_offset", 0) for s in stats
@@ -363,6 +452,7 @@ def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
 
     upload_summary = {
         "images": total_image_count,
+        # TODO: rename sequences to total uploads
         "sequences": total_uploaded_sequence_count,
         "size": round(total_entity_size_mb, 4),
         "uploaded_size": round(total_uploaded_size_mb, 4),
@@ -373,129 +463,126 @@ def _summarize(stats: T.Sequence[_APIStats]) -> T.Dict:
     return upload_summary
 
 
-def _show_upload_summary(stats: T.Sequence[_APIStats]):
-    grouped: T.Dict[str, T.List[_APIStats]] = {}
-    for stat in stats:
-        grouped.setdefault(stat.get("file_type", "unknown"), []).append(stat)
+def _show_upload_summary(stats: T.Sequence[_APIStats], errors: T.Sequence[Exception]):
+    LOG.info("==> Upload summary")
+
+    errors_by_type: dict[type[Exception], list[Exception]] = {}
+    for error in errors:
+        errors_by_type.setdefault(type(error), []).append(error)
 
-    for file_type, typed_stats in grouped.items():
-        if file_type == FileType.IMAGE.value:
+    for error_type, error_list in errors_by_type.items():
+        if error_type is UploadedAlready:
             LOG.info(
-                "%8d  %s sequences uploaded",
-                len(typed_stats),
-                file_type.upper(),
+                f"Skipped {len(error_list)} already uploaded sequences (use --reupload to force re-upload)",
             )
         else:
-            LOG.info(
-                "%8d  %s files uploaded",
-                len(typed_stats),
-                file_type.upper(),
-            )
+            LOG.info(f"{len(error_list)} uploads failed due to {error_type.__name__}")
 
-    summary = _summarize(stats)
-    LOG.info("%8.1fM data in total", summary["size"])
-    LOG.info("%8.1fM data uploaded", summary["uploaded_size"])
-    LOG.info("%8.1fs upload time", summary["time"])
+    if stats:
+        grouped: dict[str, list[_APIStats]] = {}
+        for stat in stats:
+            grouped.setdefault(stat.get("file_type", "unknown"), []).append(stat)
 
+        for file_type, typed_stats in grouped.items():
+            if file_type == FileType.IMAGE.value:
+                LOG.info(f"{len(typed_stats)} sequences uploaded")
+            else:
+                LOG.info(f"{len(typed_stats)} {file_type} uploaded")
 
-def _api_logging_finished(summary: T.Dict):
-    if MAPILLARY_DISABLE_API_LOGGING:
+        summary = _summarize(stats)
+        LOG.info(f"{humanize.naturalsize(summary['size'] * 1024 * 1024)} read in total")
+        LOG.info(
+            f"{humanize.naturalsize(summary['uploaded_size'] * 1024 * 1024)} uploaded"
+        )
+        LOG.info(f"{summary['time']} upload time")
+    else:
+        LOG.info("Nothing uploaded. Bye.")
+
+
+def _api_logging_finished(summary: dict, dry_run: bool = False):
+    if dry_run:
+        return
+
+    if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     action: api_v4.ActionType = "upload_finished_upload"
-    LOG.debug("API Logging for action %s: %s", action, summary)
     try:
         api_v4.log_event(action, summary)
     except requests.HTTPError as exc:
         LOG.warning(
-            "HTTPError from API Logging for action %s: %s",
-            action,
-            api_v4.readable_http_error(exc),
+            f"HTTPError from logging action {action}: {api_v4.readable_http_error(exc)}"
         )
     except Exception:
-        LOG.warning("Error from API Logging for action %s", action, exc_info=True)
+        LOG.warning(f"Error from logging action {action}", exc_info=True)
 
 
-def _api_logging_failed(payload: T.Dict, exc: Exception):
-    if MAPILLARY_DISABLE_API_LOGGING:
+def _api_logging_failed(payload: dict, exc: Exception, dry_run: bool = False):
+    if dry_run:
+        return
+
+    if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     payload_with_reason = {**payload, "reason": exc.__class__.__name__}
     action: api_v4.ActionType = "upload_failed_upload"
-    LOG.debug("API Logging for action %s: %s", action, payload)
     try:
         api_v4.log_event(action, payload_with_reason)
     except requests.HTTPError as exc:
         LOG.warning(
-            "HTTPError from API Logging for action %s: %s",
-            action,
-            api_v4.readable_http_error(exc),
+            f"HTTPError from logging action {action}: {api_v4.readable_http_error(exc)}"
         )
     except Exception:
-        LOG.warning("Error from API Logging for action %s", action, exc_info=True)
-
+        LOG.warning(f"Error from logging action {action}", exc_info=True)
 
-def _load_descs(
-    _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]],
-    desc_path: T.Optional[str],
-    import_paths: T.Sequence[Path],
-) -> T.List[types.Metadata]:
-    metadatas: T.List[types.Metadata]
 
-    if _metadatas_from_process is not None:
-        metadatas = [
-            metadata
-            for metadata in _metadatas_from_process
-            if not isinstance(metadata, types.ErrorMetadata)
-        ]
-    else:
-        metadatas = _load_validate_metadatas_from_desc_path(desc_path, import_paths)
+_M = T.TypeVar("_M", bound=types.Metadata)
 
-    # Make sure all metadatas have sequence uuid assigned
-    # It is used to find the right sequence when writing upload history
-    missing_sequence_uuid = str(uuid.uuid4())
-    for metadata in metadatas:
-        if isinstance(metadata, types.ImageMetadata):
-            if metadata.MAPSequenceUUID is None:
-                metadata.MAPSequenceUUID = missing_sequence_uuid
 
-    for metadata in metadatas:
-        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
-        if isinstance(metadata, types.ImageMetadata):
-            assert metadata.MAPSequenceUUID is not None
+def _find_metadata_with_filename_existed_in(
+    metadatas: T.Iterable[_M], paths: T.Iterable[Path]
+) -> list[_M]:
+    resolved_image_paths = set(p.resolve() for p in paths)
+    return [d for d in metadatas if d.filename.resolve() in resolved_image_paths]
 
-    return metadatas
 
+def _gen_upload_everything(
+    mly_uploader: uploader.Uploader,
+    metadatas: T.Sequence[types.Metadata],
+    import_paths: T.Sequence[Path],
+    skip_subfolders: bool,
+):
+    # Upload images
+    image_metadatas = _find_metadata_with_filename_existed_in(
+        (m for m in metadatas if isinstance(m, types.ImageMetadata)),
+        utils.find_images(import_paths, skip_subfolders=skip_subfolders),
+    )
+    yield from uploader.ImageSequenceUploader.upload_images(
+        mly_uploader, image_metadatas
+    )
 
-_M = T.TypeVar("_M", bound=types.Metadata)
+    # Upload videos
+    video_metadatas = _find_metadata_with_filename_existed_in(
+        (m for m in metadatas if isinstance(m, types.VideoMetadata)),
+        utils.find_videos(import_paths, skip_subfolders=skip_subfolders),
+    )
+    yield from uploader.VideoUploader.upload_videos(mly_uploader, video_metadatas)
 
+    # Upload zip files
+    zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
+    yield from uploader.ZipUploader.upload_zipfiles(mly_uploader, zip_paths)
 
-def _find_metadata_with_filename_existed_in(
-    metadatas: T.Sequence[_M], paths: T.Sequence[Path]
-) -> T.List[_M]:
-    resolved_image_paths = set(p.resolve() for p in paths)
-    return [d for d in metadatas if d.filename.resolve() in resolved_image_paths]
 
+def _normalize_import_paths(import_path: Path | T.Sequence[Path]) -> list[Path]:
+    import_paths: list[Path]
 
-def upload(
-    import_path: T.Union[Path, T.Sequence[Path]],
-    desc_path: T.Optional[str] = None,
-    _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None,
-    user_name: T.Optional[str] = None,
-    organization_key: T.Optional[str] = None,
-    dry_run=False,
-    skip_subfolders=False,
-) -> None:
-    import_paths: T.Sequence[Path]
     if isinstance(import_path, Path):
         import_paths = [import_path]
     else:
         assert isinstance(import_path, list)
         import_paths = import_path
-    import_paths = list(utils.deduplicate_paths(import_paths))
 
-    if not import_paths:
-        return
+    import_paths = list(utils.deduplicate_paths(import_paths))
 
     # Check and fail early
     for path in import_paths:
@@ -504,176 +591,113 @@ def upload(
                 f"Import file or directory not found: {path}"
             )
 
-    metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
+    return import_paths
 
-    user_items = fetch_user_items(user_name, organization_key)
 
-    # Setup the emitter -- the order matters here
+def _continue_or_fail(ex: Exception) -> Exception:
+    """
+    Wrap the exception, or re-raise if it is a fatal error (i.e. there is no point to continue)
+    """
 
-    emitter = uploader.EventEmitter()
+    if isinstance(ex, uploader.SequenceError):
+        return ex
 
-    enable_history = history.MAPILLARY_UPLOAD_HISTORY_PATH and (
-        not dry_run or MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN == "YES"
-    )
+    # Certain files not found or no permission
+    if isinstance(ex, (FileNotFoundError, PermissionError)):
+        return ex
 
-    # Put it first one to cancel early
-    if enable_history:
-        _setup_cancel_due_to_duplication(emitter)
+    # Certain metadatas are not valid
+    if isinstance(ex, exceptions.MapillaryMetadataValidationError):
+        return ex
 
-    # This one set up tdqm
-    _setup_tdqm(emitter)
+    # Fatal error: this is thrown after all retries
+    if isinstance(ex, requests.ConnectionError):
+        raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
 
-    # Now stats is empty but it will collect during upload
-    stats = _setup_api_stats(emitter)
+    # Fatal error: this is thrown after all retries
+    if isinstance(ex, requests.Timeout):
+        raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
 
-    # Send the progress as well as the log stats collected above
-    _setup_ipc(emitter)
+    # Fatal error:
+    if isinstance(ex, requests.HTTPError) and isinstance(
+        ex.response, requests.Response
+    ):
+        if api_v4.is_auth_error(ex.response):
+            raise exceptions.MapillaryUploadUnauthorizedError(
+                api_v4.extract_auth_error_message(ex.response)
+            ) from ex
+        raise ex
 
-    params: JSONDict = {
-        # null if multiple paths provided
-        "import_path": str(import_path) if isinstance(import_path, Path) else None,
-        "organization_key": user_items.get("MAPOrganizationKey"),
-        "user_key": user_items.get("MAPSettingsUserKey"),
-        "version": VERSION,
-    }
+    raise ex
 
-    if enable_history:
-        _setup_write_upload_history(emitter, params, metadatas)
 
-    mly_uploader = uploader.Uploader(
-        user_items,
-        emitter=emitter,
-        dry_run=dry_run,
-        chunk_size=int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024),
-    )
+def _load_descs(
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
+    import_paths: T.Sequence[Path],
+    desc_path: str | None,
+) -> list[types.Metadata]:
+    metadatas: list[types.Metadata]
 
-    try:
-        image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders)
-        # find descs that match the image paths from the import paths
-        image_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.ImageMetadata)
-        ]
-        specified_image_metadatas = _find_metadata_with_filename_existed_in(
-            image_metadatas, image_paths
-        )
-        if specified_image_metadatas:
-            try:
-                clusters = mly_uploader.upload_images(
-                    specified_image_metadatas,
-                    event_payload={"file_type": FileType.IMAGE.value},
-                )
-            except Exception as ex:
-                raise UploadError(ex) from ex
-
-            if clusters:
-                LOG.debug("Uploaded to cluster: %s", clusters)
-
-        video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders)
-        video_metadatas = [
-            metadata
-            for metadata in (metadatas or [])
-            if isinstance(metadata, types.VideoMetadata)
-        ]
-        specified_video_metadatas = _find_metadata_with_filename_existed_in(
-            video_metadatas, video_paths
-        )
-        for idx, video_metadata in enumerate(specified_video_metadatas):
-            video_metadata.update_md5sum()
-            assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
-
-            # extract telemetry measurements from GoPro videos
-            telemetry_measurements: T.List[telemetry.TelemetryMeasurement] = []
-            if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES":
-                if video_metadata.filetype is FileType.GOPRO:
-                    with video_metadata.filename.open("rb") as fp:
-                        telemetry_data = gpmf_parser.extract_telemetry_data(fp)
-                    if telemetry_data:
-                        telemetry_measurements.extend(telemetry_data.accl)
-                        telemetry_measurements.extend(telemetry_data.gyro)
-                        telemetry_measurements.extend(telemetry_data.magn)
-                    telemetry_measurements.sort(key=lambda m: m.time)
-
-            generator = camm_builder.camm_sample_generator2(
-                video_metadata, telemetry_measurements=telemetry_measurements
-            )
+    if _metadatas_from_process is not None:
+        metadatas, _ = types.separate_errors(_metadatas_from_process)
+    else:
+        metadatas = _load_valid_metadatas_from_desc_path(import_paths, desc_path)
 
-            with video_metadata.filename.open("rb") as src_fp:
-                camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator)
-                event_payload: uploader.Progress = {
-                    "total_sequence_count": len(specified_video_metadatas),
-                    "sequence_idx": idx,
-                    "file_type": video_metadata.filetype.value,
-                    "import_path": str(video_metadata.filename),
-                }
-                try:
-                    cluster_id = mly_uploader.upload_stream(
-                        T.cast(T.BinaryIO, camm_fp),
-                        upload_api_v4.ClusterFileType.CAMM,
-                        video_metadata.md5sum,
-                        event_payload=event_payload,
-                    )
-                except Exception as ex:
-                    raise UploadError(ex) from ex
-                LOG.debug("Uploaded to cluster: %s", cluster_id)
-
-        zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
-        _upload_zipfiles(mly_uploader, zip_paths)
-
-    except UploadError as ex:
-        inner_ex = ex.inner_ex
-
-        if not dry_run:
-            _api_logging_failed(_summarize(stats), inner_ex)
-
-        if isinstance(inner_ex, requests.ConnectionError):
-            raise exceptions.MapillaryUploadConnectionError(str(inner_ex)) from inner_ex
-
-        if isinstance(inner_ex, requests.Timeout):
-            raise exceptions.MapillaryUploadTimeoutError(str(inner_ex)) from inner_ex
-
-        if isinstance(inner_ex, requests.HTTPError) and isinstance(
-            inner_ex.response, requests.Response
-        ):
-            if inner_ex.response.status_code in [400, 401]:
-                try:
-                    error_body = inner_ex.response.json()
-                except Exception:
-                    error_body = {}
-                debug_info = error_body.get("debug_info", {})
-                if debug_info.get("type") in ["NotAuthorizedError"]:
-                    raise exceptions.MapillaryUploadUnauthorizedError(
-                        debug_info.get("message")
-                    ) from inner_ex
-            raise inner_ex
-
-        raise inner_ex
+    # Make sure all metadatas have sequence uuid assigned
+    # It is used to find the right sequence when writing upload history
+    missing_sequence_uuid = str(uuid.uuid4())
+    for metadata in metadatas:
+        if isinstance(metadata, types.ImageMetadata):
+            if metadata.MAPSequenceUUID is None:
+                metadata.MAPSequenceUUID = missing_sequence_uuid
 
-    if stats:
-        if not dry_run:
-            _api_logging_finished(_summarize(stats))
-        _show_upload_summary(stats)
-    else:
-        LOG.info("Nothing uploaded. Bye.")
+    for metadata in metadatas:
+        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
+        if isinstance(metadata, types.ImageMetadata):
+            assert metadata.MAPSequenceUUID is not None
 
+    return metadatas
 
-def _upload_zipfiles(
-    mly_uploader: uploader.Uploader,
-    zip_paths: T.Sequence[Path],
-) -> None:
-    for idx, zip_path in enumerate(zip_paths):
-        event_payload: uploader.Progress = {
-            "total_sequence_count": len(zip_paths),
-            "sequence_idx": idx,
-            "file_type": FileType.ZIP.value,
-            "import_path": str(zip_path),
-        }
+
+def _load_valid_metadatas_from_desc_path(
+    import_paths: T.Sequence[Path], desc_path: str | None
+) -> list[types.Metadata]:
+    if desc_path is None:
+        desc_path = _find_desc_path(import_paths)
+
+    if desc_path == "-":
         try:
-            cluster_id = mly_uploader.upload_zipfile(
-                zip_path, event_payload=event_payload
+            metadatas = DescriptionJSONSerializer.deserialize_stream(sys.stdin.buffer)
+        except json.JSONDecodeError as ex:
+            raise exceptions.MapillaryInvalidDescriptionFile(
+                f"Invalid JSON stream from {desc_path}: {ex}"
+            ) from ex
+
+    else:
+        if not os.path.isfile(desc_path):
+            raise exceptions.MapillaryFileNotFoundError(
+                f"Description file not found: {desc_path}"
             )
-        except Exception as ex:
-            raise UploadError(ex) from ex
+        with open(desc_path, "rb") as fp:
+            try:
+                metadatas = DescriptionJSONSerializer.deserialize_stream(fp)
+            except json.JSONDecodeError as ex:
+                raise exceptions.MapillaryInvalidDescriptionFile(
+                    f"Invalid JSON stream from {desc_path}: {ex}"
+                ) from ex
 
-        LOG.debug("Uploaded to cluster: %s", cluster_id)
+    return metadatas
+
+
+def _find_desc_path(import_paths: T.Sequence[Path]) -> str:
+    if len(import_paths) == 1 and import_paths[0].is_dir():
+        return str(import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME))
+
+    if 1 < len(import_paths):
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading multiple paths"
+        )
+    else:
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading a single file"
+        )