mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.1__py3-none-any.whl

mapillary_tools/upload.py

@@ -9,27 +9,25 @@ import typing as T
 import uuid
 from pathlib import Path
 
+import humanize
 import jsonschema
 import requests
 from tqdm import tqdm
 
 from . import (
     api_v4,
+    config,
     constants,
     exceptions,
-    geo,
     history,
+    http,
     ipc,
-    telemetry,
     types,
-    upload_api_v4,
     uploader,
     utils,
     VERSION,
 )
-from .camm import camm_builder, camm_parser
-from .gpmf import gpmf_parser
-from .mp4 import simple_mp4_builder
+from .serializer.description import DescriptionJSONSerializer
 from .types import FileType
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
@@ -37,91 +35,110 @@ JSONDict = T.Dict[str, T.Union[str, int, float, None]]
 LOG = logging.getLogger(__name__)
 
 
-class UploadedAlreadyError(uploader.SequenceError):
+class UploadedAlready(uploader.SequenceError):
     pass
 
 
-def _load_validate_metadatas_from_desc_path(
-    desc_path: str | None, import_paths: T.Sequence[Path]
-) -> list[types.Metadata]:
-    is_default_desc_path = False
-    if desc_path is None:
-        is_default_desc_path = True
-        if len(import_paths) == 1 and import_paths[0].is_dir():
-            desc_path = str(
-                import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME)
-            )
-        else:
-            if 1 < len(import_paths):
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading multiple paths",
-                )
-            else:
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading a single file",
-                )
+def upload(
+    import_path: Path | T.Sequence[Path],
+    user_items: config.UserItem,
+    num_upload_workers: int,
+    desc_path: str | None = None,
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
+    reupload: bool = False,
+    dry_run: bool = False,
+    nofinish: bool = False,
+    noresume: bool = False,
+    skip_subfolders: bool = False,
+) -> None:
+    LOG.info("==> Uploading...")
 
-    descs: list[types.DescriptionOrError] = []
+    import_paths = _normalize_import_paths(import_path)
 
-    if desc_path == "-":
-        try:
-            descs = json.load(sys.stdin)
-        except json.JSONDecodeError as ex:
-            raise exceptions.MapillaryInvalidDescriptionFile(
-                f"Invalid JSON stream from stdin: {ex}"
-            ) from ex
-    else:
-        if not os.path.isfile(desc_path):
-            if is_default_desc_path:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found. Has the directory been processed yet?"
-                )
-            else:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found"
-                )
-        with open(desc_path) as fp:
-            try:
-                descs = json.load(fp)
-            except json.JSONDecodeError as ex:
-                raise exceptions.MapillaryInvalidDescriptionFile(
-                    f"Invalid JSON file {desc_path}: {ex}"
-                ) from ex
+    metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
 
-    # the descs load from stdin or json file may contain invalid entries
-    validated_descs = [
-        types.validate_and_fail_desc(desc)
-        for desc in descs
-        # skip error descriptions
-        if "error" not in desc
-    ]
+    jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
+
+    # Setup the emitter -- the order matters here
+
+    emitter = uploader.EventEmitter()
 
-    # throw if we found any invalid descs
-    invalid_descs = [desc for desc in validated_descs if "error" in desc]
-    if invalid_descs:
-        for desc in invalid_descs:
-            LOG.error("Invalid description entry: %s", json.dumps(desc))
-        raise exceptions.MapillaryInvalidDescriptionFile(
-            f"Found {len(invalid_descs)} invalid descriptions"
+    # Check duplications first
+    if not _is_history_disabled(dry_run):
+        upload_run_params: JSONDict = {
+            # Null if multiple paths provided
+            "import_path": str(import_path) if isinstance(import_path, Path) else None,
+            "organization_key": user_items.get("MAPOrganizationKey"),
+            "user_key": user_items.get("MAPSettingsUserKey"),
+            "version": VERSION,
+            "run_at": time.time(),
+        }
+        _setup_history(
+            emitter, upload_run_params, metadatas, reupload=reupload, nofinish=nofinish
         )
 
-    # validated_descs should contain no errors
-    return [
-        types.from_desc(T.cast(types.Description, desc)) for desc in validated_descs
-    ]
+    # Set up tdqm
+    _setup_tdqm(emitter)
 
+    # Now stats is empty but it will collect during ALL uploads
+    stats = _setup_api_stats(emitter)
 
-def zip_images(
-    import_path: Path,
-    zip_dir: Path,
-    desc_path: str | None = None,
-):
+    # Send the progress via IPC, and log the progress in debug mode
+    _setup_ipc(emitter)
+
+    try:
+        upload_options = uploader.UploadOptions(
+            user_items,
+            dry_run=dry_run,
+            nofinish=nofinish,
+            noresume=noresume,
+            num_upload_workers=num_upload_workers,
+        )
+    except ValueError as ex:
+        raise exceptions.MapillaryBadParameterError(str(ex)) from ex
+
+    mly_uploader = uploader.Uploader(upload_options, emitter=emitter)
+
+    results = _gen_upload_everything(
+        mly_uploader, metadatas, import_paths, skip_subfolders
+    )
+
+    upload_successes = 0
+    upload_errors: list[Exception] = []
+
+    # The real uploading happens sequentially here
+    try:
+        for _, result in results:
+            if result.error is not None:
+                upload_error = _continue_or_fail(result.error)
+                log_exception(upload_error)
+                upload_errors.append(upload_error)
+            else:
+                upload_successes += 1
+
+    except Exception as ex:
+        # Fatal error: log and raise
+        _api_logging_failed(_summarize(stats), ex, dry_run=dry_run)
+        raise ex
+
+    else:
+        _api_logging_finished(_summarize(stats), dry_run=dry_run)
+
+    finally:
+        # We collected stats after every upload is finished
+        assert upload_successes == len(stats), (
+            f"Expect {upload_successes} success but got {stats}"
+        )
+        _show_upload_summary(stats, upload_errors)
+
+
+def zip_images(import_path: Path, zip_dir: Path, desc_path: str | None = None):
     if not import_path.is_dir():
         raise exceptions.MapillaryFileNotFoundError(
             f"Import directory not found: {import_path}"
         )
 
-    metadatas = _load_validate_metadatas_from_desc_path(desc_path, [import_path])
+    metadatas = _load_valid_metadatas_from_desc_path([import_path], desc_path)
 
     if not metadatas:
         LOG.warning("No images or videos found in %s", desc_path)
@@ -131,40 +148,90 @@ def zip_images(
         metadata for metadata in metadatas if isinstance(metadata, types.ImageMetadata)
     ]
 
-    uploader.ZipImageSequence.zip_images(image_metadatas, zip_dir)
+    uploader.ZipUploader.zip_images(image_metadatas, zip_dir)
+
+
+def log_exception(ex: Exception) -> None:
+    if LOG.isEnabledFor(logging.DEBUG):
+        exc_info = ex
+    else:
+        exc_info = None
+
+    exc_name = ex.__class__.__name__
+
+    if isinstance(ex, UploadedAlready):
+        LOG.info(f"{exc_name}: {ex}")
+    elif isinstance(ex, requests.HTTPError):
+        LOG.error(f"{exc_name}: {http.readable_http_error(ex)}", exc_info=exc_info)
+    elif isinstance(ex, api_v4.HTTPContentError):
+        LOG.error(
+            f"{exc_name}: {ex}: {http.readable_http_response(ex.response)}",
+            exc_info=exc_info,
+        )
+    else:
+        LOG.error(f"{exc_name}: {ex}", exc_info=exc_info)
+
+
+def _is_history_disabled(dry_run: bool) -> bool:
+    # There is no way to read/write history if the path is not set
+    if not constants.MAPILLARY_UPLOAD_HISTORY_PATH:
+        return True
+
+    if dry_run:
+        # When dry_run mode is on, we disable history by default
+        # However, we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
+        # and when it is on, we enable history regardless of dry_run
+        if constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN:
+            return False
+        else:
+            return True
+
+    return False
 
 
 def _setup_history(
     emitter: uploader.EventEmitter,
     upload_run_params: JSONDict,
     metadatas: list[types.Metadata],
+    reupload: bool,
+    nofinish: bool,
 ) -> None:
     @emitter.on("upload_start")
     def check_duplication(payload: uploader.Progress):
-        md5sum = payload.get("md5sum")
+        md5sum = payload.get("sequence_md5sum")
         assert md5sum is not None, f"md5sum has to be set for {payload}"
 
-        if history.is_uploaded(md5sum):
-            sequence_uuid = payload.get("sequence_uuid")
-            if sequence_uuid is None:
-                basename = os.path.basename(payload.get("import_path", ""))
-                LOG.info(
-                    "File %s has been uploaded already. Check the upload history at %s",
-                    basename,
-                    history.history_desc_path(md5sum),
-                )
+        record = history.read_history_record(md5sum)
+
+        if record is not None:
+            history_desc_path = history.history_desc_path(md5sum)
+            uploaded_at = record.get("summary", {}).get("upload_end_time", None)
+
+            upload_name = uploader.Uploader._upload_name(payload)
+
+            if reupload:
+                if uploaded_at is not None:
+                    LOG.info(
+                        f"Reuploading {upload_name}, despite being uploaded {humanize.naturaldelta(time.time() - uploaded_at)} ago ({time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(uploaded_at))})"
+                    )
+                else:
+                    LOG.info(
+                        f"Reuploading {upload_name}, despite already being uploaded (see {history_desc_path})"
+                    )
             else:
-                LOG.info(
-                    "Sequence %s has been uploaded already. Check the upload history at %s",
-                    sequence_uuid,
-                    history.history_desc_path(md5sum),
-                )
-            raise UploadedAlreadyError()
+                if uploaded_at is not None:
+                    msg = f"Skipping {upload_name}, already uploaded {humanize.naturaldelta(time.time() - uploaded_at)} ago ({time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(uploaded_at))})"
+                else:
+                    msg = f"Skipping {upload_name}, already uploaded (see {history_desc_path})"
+                raise UploadedAlready(msg)
 
     @emitter.on("upload_finished")
     def write_history(payload: uploader.Progress):
+        if nofinish:
+            return
+
         sequence_uuid = payload.get("sequence_uuid")
-        md5sum = payload.get("md5sum")
+        md5sum = payload.get("sequence_md5sum")
         assert md5sum is not None, f"md5sum has to be set for {payload}"
 
         if sequence_uuid is None:
@@ -180,10 +247,7 @@ def _setup_history(
 
         try:
             history.write_history(
-                md5sum,
-                upload_run_params,
-                T.cast(JSONDict, payload),
-                sequence,
+                md5sum, upload_run_params, T.cast(JSONDict, payload), sequence
             )
         except OSError:
             LOG.warning("Error writing upload history %s", md5sum, exc_info=True)
@@ -192,8 +256,8 @@ def _setup_history(
 def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
     upload_pbar: tqdm | None = None
 
-    @emitter.on("upload_fetch_offset")
-    def upload_fetch_offset(payload: uploader.Progress) -> None:
+    @emitter.on("upload_start")
+    def upload_start(payload: uploader.Progress) -> None:
         nonlocal upload_pbar
 
         if upload_pbar is not None:
@@ -204,27 +268,52 @@ def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
         import_path: str | None = payload.get("import_path")
         filetype = payload.get("file_type", "unknown").upper()
         if import_path is None:
-            _desc = f"Uploading {filetype} ({nth}/{total})"
+            desc = f"Uploading {filetype} ({nth}/{total})"
         else:
-            _desc = (
+            desc = (
                 f"Uploading {filetype} {os.path.basename(import_path)} ({nth}/{total})"
             )
         upload_pbar = tqdm(
             total=payload["entity_size"],
-            desc=_desc,
+            desc=desc,
             unit="B",
             unit_scale=True,
             unit_divisor=1024,
-            initial=payload["offset"],
-            disable=LOG.getEffectiveLevel() <= logging.DEBUG,
+            initial=payload.get("offset", 0),
+            disable=LOG.isEnabledFor(logging.DEBUG),
+        )
+
+    @emitter.on("upload_fetch_offset")
+    def upload_fetch_offset(payload: uploader.Progress) -> None:
+        assert upload_pbar is not None, (
+            "progress_bar must be initialized in upload_start"
         )
+        begin_offset = payload.get("begin_offset", 0)
+        if begin_offset is not None and begin_offset > 0:
+            if upload_pbar.total is not None:
+                progress_percent = (begin_offset / upload_pbar.total) * 100
+                upload_pbar.write(
+                    f"Resuming upload at {begin_offset=} ({progress_percent:3.0f}%)",
+                    file=sys.stderr,
+                )
+            else:
+                upload_pbar.write(
+                    f"Resuming upload at {begin_offset=}", file=sys.stderr
+                )
+            upload_pbar.reset()
+            upload_pbar.update(begin_offset)
+            upload_pbar.refresh()
 
     @emitter.on("upload_progress")
     def upload_progress(payload: uploader.Progress) -> None:
-        assert upload_pbar is not None, "progress_bar must be initialized"
+        assert upload_pbar is not None, (
+            "progress_bar must be initialized in upload_start"
+        )
         upload_pbar.update(payload["chunk_size"])
+        upload_pbar.refresh()
 
     @emitter.on("upload_end")
+    @emitter.on("upload_failed")
     def upload_end(_: uploader.Progress) -> None:
         nonlocal upload_pbar
         if upload_pbar:
@@ -236,20 +325,20 @@ def _setup_ipc(emitter: uploader.EventEmitter):
     @emitter.on("upload_start")
     def upload_start(payload: uploader.Progress):
         type: uploader.EventName = "upload_start"
-        LOG.debug("IPC %s: %s", type.upper(), payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
     @emitter.on("upload_fetch_offset")
     def upload_fetch_offset(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_fetch_offset"
-        LOG.debug("IPC %s: %s", type.upper(), payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
     @emitter.on("upload_progress")
     def upload_progress(payload: uploader.Progress):
         type: uploader.EventName = "upload_progress"
 
-        if LOG.getEffectiveLevel() <= logging.DEBUG:
+        if LOG.isEnabledFor(logging.DEBUG):
             # In debug mode, we want to see the progress every 30 seconds
             # instead of every chunk (which is too verbose)
             INTERVAL_SECONDS = 30
@@ -261,7 +350,7 @@ def _setup_ipc(emitter: uploader.EventEmitter):
                 last_upload_progress_debug_at is None
                 or last_upload_progress_debug_at + INTERVAL_SECONDS < now
             ):
-                LOG.debug("IPC %s: %s", type.upper(), payload)
+                LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
                 T.cast(T.Dict, payload)["_last_upload_progress_debug_at"] = now
 
         ipc.send(type, payload)
@@ -269,7 +358,13 @@ def _setup_ipc(emitter: uploader.EventEmitter):
     @emitter.on("upload_end")
     def upload_end(payload: uploader.Progress) -> None:
         type: uploader.EventName = "upload_end"
-        LOG.debug("IPC %s: %s", type.upper(), payload)
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
+        ipc.send(type, payload)
+
+    @emitter.on("upload_failed")
+    def upload_failed(payload: uploader.Progress) -> None:
+        type: uploader.EventName = "upload_failed"
+        LOG.debug(f"{type.upper()}: {json.dumps(payload)}")
         ipc.send(type, payload)
 
 
@@ -295,8 +390,13 @@ def _setup_api_stats(emitter: uploader.EventEmitter):
 
     @emitter.on("upload_start")
     def collect_start_time(payload: _APIStats) -> None:
-        payload["upload_start_time"] = time.time()
+        now = time.time()
+        payload["upload_start_time"] = now
         payload["upload_total_time"] = 0
+        # These filed should be initialized in upload events like "upload_fetch_offset"
+        # but since we disabled them for uploading images, so we initialize them here
+        payload["upload_last_restart_time"] = now
+        payload["upload_first_offset"] = 0
 
     @emitter.on("upload_fetch_offset")
     def collect_restart_time(payload: _APIStats) -> None:
@@ -305,8 +405,8 @@ def _setup_api_stats(emitter: uploader.EventEmitter):
             payload["offset"], payload.get("upload_first_offset", payload["offset"])
         )
 
-    @emitter.on("upload_interrupted")
-    def collect_interrupted(payload: _APIStats):
+    @emitter.on("upload_retrying")
+    def collect_retrying(payload: _APIStats):
         # could be None if it failed to fetch offset
         restart_time = payload.get("upload_last_restart_time")
         if restart_time is not None:
@@ -360,89 +460,80 @@ def _summarize(stats: T.Sequence[_APIStats]) -> dict:
 
 
 def _show_upload_summary(stats: T.Sequence[_APIStats], errors: T.Sequence[Exception]):
-    if not stats:
-        LOG.info("Nothing uploaded. Bye.")
-    else:
+    LOG.info("==> Upload summary")
+
+    errors_by_type: dict[type[Exception], list[Exception]] = {}
+    for error in errors:
+        errors_by_type.setdefault(type(error), []).append(error)
+
+    for error_type, error_list in errors_by_type.items():
+        if error_type is UploadedAlready:
+            LOG.info(
+                f"Skipped {len(error_list)} already uploaded sequences (use --reupload to force re-upload)",
+            )
+        else:
+            LOG.info(f"{len(error_list)} uploads failed due to {error_type.__name__}")
+
+    if stats:
         grouped: dict[str, list[_APIStats]] = {}
         for stat in stats:
             grouped.setdefault(stat.get("file_type", "unknown"), []).append(stat)
 
         for file_type, typed_stats in grouped.items():
             if file_type == FileType.IMAGE.value:
-                LOG.info("%8d image sequences uploaded", len(typed_stats))
+                LOG.info(f"{len(typed_stats)} sequences uploaded")
             else:
-                LOG.info("%8d  %s videos uploaded", len(typed_stats), file_type.upper())
+                LOG.info(f"{len(typed_stats)} {file_type} uploaded")
 
         summary = _summarize(stats)
-        LOG.info("%8.1fM data in total", summary["size"])
-        LOG.info("%8.1fM data uploaded", summary["uploaded_size"])
-        LOG.info("%8.1fs upload time", summary["time"])
+        LOG.info(f"{humanize.naturalsize(summary['size'] * 1024 * 1024)} read in total")
+        LOG.info(
+            f"{humanize.naturalsize(summary['uploaded_size'] * 1024 * 1024)} uploaded"
+        )
+        LOG.info(f"{summary['time']:.3f} seconds upload time")
+    else:
+        LOG.info("Nothing uploaded. Bye.")
 
-    for error in errors:
-        LOG.error("Upload error: %s: %s", error.__class__.__name__, error)
 
+def _api_logging_finished(summary: dict, dry_run: bool = False):
+    if dry_run:
+        return
 
-def _api_logging_finished(summary: dict):
     if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     action: api_v4.ActionType = "upload_finished_upload"
-    try:
-        api_v4.log_event(action, summary)
-    except requests.HTTPError as exc:
-        LOG.warning(
-            "HTTPError from API Logging for action %s: %s",
-            action,
-            api_v4.readable_http_error(exc),
-        )
-    except Exception:
-        LOG.warning("Error from API Logging for action %s", action, exc_info=True)
 
+    with api_v4.create_client_session(disable_logging=True) as client_session:
+        try:
+            api_v4.log_event(client_session, action, summary)
+        except requests.HTTPError as exc:
+            LOG.warning(
+                f"HTTPError from logging action {action}: {http.readable_http_error(exc)}"
+            )
+        except Exception:
+            LOG.warning(f"Error from logging action {action}", exc_info=True)
+
+
+def _api_logging_failed(payload: dict, exc: Exception, dry_run: bool = False):
+    if dry_run:
+        return
 
-def _api_logging_failed(payload: dict, exc: Exception):
     if constants.MAPILLARY_DISABLE_API_LOGGING:
         return
 
     payload_with_reason = {**payload, "reason": exc.__class__.__name__}
     action: api_v4.ActionType = "upload_failed_upload"
-    try:
-        api_v4.log_event(action, payload_with_reason)
-    except requests.HTTPError as exc:
-        LOG.warning(
-            "HTTPError from API Logging for action %s: %s",
-            action,
-            api_v4.readable_http_error(exc),
-        )
-    except Exception:
-        LOG.warning("Error from API Logging for action %s", action, exc_info=True)
 
-
-def _load_descs(
-    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
-    desc_path: str | None,
-    import_paths: T.Sequence[Path],
-) -> list[types.Metadata]:
-    metadatas: list[types.Metadata]
-
-    if _metadatas_from_process is not None:
-        metadatas, _ = types.separate_errors(_metadatas_from_process)
-    else:
-        metadatas = _load_validate_metadatas_from_desc_path(desc_path, import_paths)
-
-    # Make sure all metadatas have sequence uuid assigned
-    # It is used to find the right sequence when writing upload history
-    missing_sequence_uuid = str(uuid.uuid4())
-    for metadata in metadatas:
-        if isinstance(metadata, types.ImageMetadata):
-            if metadata.MAPSequenceUUID is None:
-                metadata.MAPSequenceUUID = missing_sequence_uuid
-
-    for metadata in metadatas:
-        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
-        if isinstance(metadata, types.ImageMetadata):
-            assert metadata.MAPSequenceUUID is not None
-
-    return metadatas
+    with api_v4.create_client_session(disable_logging=True) as client_session:
+        try:
+            api_v4.log_event(client_session, action, payload_with_reason)
+        except requests.HTTPError as exc:
+            LOG.warning(
+                f"HTTPError from logging action {action}: {http.readable_http_error(exc)}"
+            )
+        except Exception:
+            LOG.warning(f"Error from logging action {action}", exc_info=True)
 
 
 _M = T.TypeVar("_M", bound=types.Metadata)
@@ -466,110 +557,21 @@ def _gen_upload_everything(
         (m for m in metadatas if isinstance(m, types.ImageMetadata)),
         utils.find_images(import_paths, skip_subfolders=skip_subfolders),
     )
-    for image_result in uploader.ZipImageSequence.prepare_images_and_upload(
-        image_metadatas,
-        mly_uploader,
-    ):
-        yield image_result
+    image_uploader = uploader.ImageSequenceUploader(
+        mly_uploader.upload_options, emitter=mly_uploader.emitter
+    )
+    yield from image_uploader.upload_images(image_metadatas)
 
     # Upload videos
     video_metadatas = _find_metadata_with_filename_existed_in(
         (m for m in metadatas if isinstance(m, types.VideoMetadata)),
         utils.find_videos(import_paths, skip_subfolders=skip_subfolders),
     )
-    for video_result in _gen_upload_videos(mly_uploader, video_metadatas):
-        yield video_result
+    yield from uploader.VideoUploader.upload_videos(mly_uploader, video_metadatas)
 
     # Upload zip files
     zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders)
-    for zip_result in _gen_upload_zipfiles(mly_uploader, zip_paths):
-        yield zip_result
-
-
-def _gen_upload_videos(
-    mly_uploader: uploader.Uploader, video_metadatas: T.Sequence[types.VideoMetadata]
-) -> T.Generator[tuple[types.VideoMetadata, uploader.UploadResult], None, None]:
-    for idx, video_metadata in enumerate(video_metadatas):
-        try:
-            video_metadata.update_md5sum()
-        except Exception as ex:
-            yield video_metadata, uploader.UploadResult(error=ex)
-            continue
-
-        assert isinstance(video_metadata.md5sum, str), "md5sum should be updated"
-
-        # Convert video metadata to CAMMInfo
-        camm_info = _prepare_camm_info(video_metadata)
-
-        # Create the CAMM sample generator
-        camm_sample_generator = camm_builder.camm_sample_generator2(camm_info)
-
-        progress: uploader.SequenceProgress = {
-            "total_sequence_count": len(video_metadatas),
-            "sequence_idx": idx,
-            "file_type": video_metadata.filetype.value,
-            "import_path": str(video_metadata.filename),
-            "md5sum": video_metadata.md5sum,
-        }
-
-        session_key = uploader._session_key(
-            video_metadata.md5sum, upload_api_v4.ClusterFileType.CAMM
-        )
-
-        try:
-            with video_metadata.filename.open("rb") as src_fp:
-                # Build the mp4 stream with the CAMM samples
-                camm_fp = simple_mp4_builder.transform_mp4(
-                    src_fp, camm_sample_generator
-                )
-
-                # Upload the mp4 stream
-                cluster_id = mly_uploader.upload_stream(
-                    T.cast(T.IO[bytes], camm_fp),
-                    upload_api_v4.ClusterFileType.CAMM,
-                    session_key,
-                    progress=T.cast(T.Dict[str, T.Any], progress),
-                )
-        except Exception as ex:
-            yield video_metadata, uploader.UploadResult(error=ex)
-        else:
-            yield video_metadata, uploader.UploadResult(result=cluster_id)
-
-
-def _prepare_camm_info(video_metadata: types.VideoMetadata) -> camm_parser.CAMMInfo:
-    camm_info = camm_parser.CAMMInfo(
-        make=video_metadata.make or "", model=video_metadata.model or ""
-    )
-
-    for point in video_metadata.points:
-        if isinstance(point, telemetry.CAMMGPSPoint):
-            if camm_info.gps is None:
-                camm_info.gps = []
-            camm_info.gps.append(point)
-
-        elif isinstance(point, telemetry.GPSPoint):
-            # There is no proper CAMM entry for GoPro GPS
-            if camm_info.mini_gps is None:
-                camm_info.mini_gps = []
-            camm_info.mini_gps.append(point)
-
-        elif isinstance(point, geo.Point):
-            if camm_info.mini_gps is None:
-                camm_info.mini_gps = []
-            camm_info.mini_gps.append(point)
-        else:
-            raise ValueError(f"Unknown point type: {point}")
-
-    if constants.MAPILLARY__EXPERIMENTAL_ENABLE_IMU:
-        if video_metadata.filetype is FileType.GOPRO:
-            with video_metadata.filename.open("rb") as fp:
-                gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True)
-            if gopro_info is not None:
-                camm_info.accl = gopro_info.accl or []
-                camm_info.gyro = gopro_info.gyro or []
-                camm_info.magn = gopro_info.magn or []
-
-    return camm_info
+    yield from uploader.ZipUploader.upload_zipfiles(mly_uploader, zip_paths)
 
 
 def _normalize_import_paths(import_path: Path | T.Sequence[Path]) -> list[Path]:
@@ -602,7 +604,7 @@ def _continue_or_fail(ex: Exception) -> Exception:
         return ex
 
     # Certain files not found or no permission
-    if isinstance(ex, OSError):
+    if isinstance(ex, (FileNotFoundError, PermissionError)):
         return ex
 
     # Certain metadatas are not valid
@@ -630,100 +632,73 @@ def _continue_or_fail(ex: Exception) -> Exception:
     raise ex
 
 
-def upload(
-    import_path: Path | T.Sequence[Path],
-    user_items: types.UserItem,
-    desc_path: str | None = None,
-    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
-    dry_run=False,
-    skip_subfolders=False,
-) -> None:
-    import_paths = _normalize_import_paths(import_path)
-
-    metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
-
-    jsonschema.validate(instance=user_items, schema=types.UserItemSchema)
-
-    # Setup the emitter -- the order matters here
+def _load_descs(
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
+    import_paths: T.Sequence[Path],
+    desc_path: str | None,
+) -> list[types.Metadata]:
+    metadatas: list[types.Metadata]
 
-    emitter = uploader.EventEmitter()
+    if _metadatas_from_process is not None:
+        metadatas, _ = types.separate_errors(_metadatas_from_process)
+    else:
+        metadatas = _load_valid_metadatas_from_desc_path(import_paths, desc_path)
 
-    # When dry_run mode is on, we disable history by default.
-    # But we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
-    # and when it is on, we enable history regardless of dry_run
-    enable_history = constants.MAPILLARY_UPLOAD_HISTORY_PATH and (
-        not dry_run or constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
-    )
+    # Make sure all metadatas have sequence uuid assigned
+    # It is used to find the right sequence when writing upload history
+    missing_sequence_uuid = str(uuid.uuid4())
+    for metadata in metadatas:
+        if isinstance(metadata, types.ImageMetadata):
+            if metadata.MAPSequenceUUID is None:
+                metadata.MAPSequenceUUID = missing_sequence_uuid
 
-    # Put it first one to check duplications first
-    if enable_history:
-        upload_run_params: JSONDict = {
-            # Null if multiple paths provided
-            "import_path": str(import_path) if isinstance(import_path, Path) else None,
-            "organization_key": user_items.get("MAPOrganizationKey"),
-            "user_key": user_items.get("MAPSettingsUserKey"),
-            "version": VERSION,
-            "run_at": time.time(),
-        }
-        _setup_history(emitter, upload_run_params, metadatas)
+    for metadata in metadatas:
+        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
+        if isinstance(metadata, types.ImageMetadata):
+            assert metadata.MAPSequenceUUID is not None
 
-    # Set up tdqm
-    _setup_tdqm(emitter)
+    return metadatas
 
-    # Now stats is empty but it will collect during ALL uploads
-    stats = _setup_api_stats(emitter)
 
-    # Send the progress via IPC, and log the progress in debug mode
-    _setup_ipc(emitter)
+def _load_valid_metadatas_from_desc_path(
+    import_paths: T.Sequence[Path], desc_path: str | None
+) -> list[types.Metadata]:
+    if desc_path is None:
+        desc_path = _find_desc_path(import_paths)
 
-    mly_uploader = uploader.Uploader(user_items, emitter=emitter, dry_run=dry_run)
+    if desc_path == "-":
+        try:
+            metadatas = DescriptionJSONSerializer.deserialize_stream(sys.stdin.buffer)
+        except json.JSONDecodeError as ex:
+            raise exceptions.MapillaryInvalidDescriptionFile(
+                f"Invalid JSON stream from {desc_path}: {ex}"
+            ) from ex
 
-    results = _gen_upload_everything(
-        mly_uploader, metadatas, import_paths, skip_subfolders
-    )
+    else:
+        if not os.path.isfile(desc_path):
+            raise exceptions.MapillaryFileNotFoundError(
+                f"Description file not found: {desc_path}"
+            )
+        with open(desc_path, "rb") as fp:
+            try:
+                metadatas = DescriptionJSONSerializer.deserialize_stream(fp)
+            except json.JSONDecodeError as ex:
+                raise exceptions.MapillaryInvalidDescriptionFile(
+                    f"Invalid JSON stream from {desc_path}: {ex}"
+                ) from ex
 
-    upload_successes = 0
-    upload_errors: list[Exception] = []
+    return metadatas
 
-    # The real upload happens sequentially here
-    try:
-        for _, result in results:
-            if result.error is not None:
-                upload_errors.append(_continue_or_fail(result.error))
-            else:
-                upload_successes += 1
 
-    except Exception as ex:
-        # Fatal error: log and raise
-        if not dry_run:
-            _api_logging_failed(_summarize(stats), ex)
-        raise ex
+def _find_desc_path(import_paths: T.Sequence[Path]) -> str:
+    if len(import_paths) == 1 and import_paths[0].is_dir():
+        return str(import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME))
 
+    if 1 < len(import_paths):
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading multiple paths"
+        )
     else:
-        if not dry_run:
-            _api_logging_finished(_summarize(stats))
-
-    finally:
-        # We collected stats after every upload is finished
-        assert upload_successes == len(stats)
-        _show_upload_summary(stats, upload_errors)
-
-
-def _gen_upload_zipfiles(
-    mly_uploader: uploader.Uploader,
-    zip_paths: T.Sequence[Path],
-) -> T.Generator[tuple[Path, uploader.UploadResult], None, None]:
-    for idx, zip_path in enumerate(zip_paths):
-        progress: uploader.SequenceProgress = {
-            "total_sequence_count": len(zip_paths),
-            "sequence_idx": idx,
-            "import_path": str(zip_path),
-        }
-        try:
-            cluster_id = uploader.ZipImageSequence.prepare_zipfile_and_upload(
-                zip_path, mly_uploader, progress=T.cast(T.Dict[str, T.Any], progress)
-            )
-        except Exception as ex:
-            yield zip_path, uploader.UploadResult(error=ex)
-        else:
-            yield zip_path, uploader.UploadResult(result=cluster_id)
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading a single file"
+        )