mapillary-tools 0.14.0a2__py3-none-any.whl → 0.14.0b1__py3-none-any.whl

mapillary_tools/upload.py

@@ -15,6 +15,7 @@ from tqdm import tqdm
 
 from . import (
     api_v4,
+    config,
     constants,
     exceptions,
     geo,
@@ -22,7 +23,6 @@ from . import (
     ipc,
     telemetry,
     types,
-    upload_api_v4,
     uploader,
     utils,
     VERSION,
@@ -30,6 +30,7 @@ from . import (
 from .camm import camm_builder, camm_parser
 from .gpmf import gpmf_parser
 from .mp4 import simple_mp4_builder
+from .serializer.description import DescriptionJSONSerializer
 from .types import FileType
 
 JSONDict = T.Dict[str, T.Union[str, int, float, None]]
@@ -41,87 +42,94 @@ class UploadedAlreadyError(uploader.SequenceError):
     pass
 
 
-def _load_validate_metadatas_from_desc_path(
-    desc_path: str | None, import_paths: T.Sequence[Path]
-) -> list[types.Metadata]:
-    is_default_desc_path = False
-    if desc_path is None:
-        is_default_desc_path = True
-        if len(import_paths) == 1 and import_paths[0].is_dir():
-            desc_path = str(
-                import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME)
-            )
-        else:
-            if 1 < len(import_paths):
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading multiple paths",
-                )
-            else:
-                raise exceptions.MapillaryBadParameterError(
-                    "The description path must be specified (with --desc_path) when uploading a single file",
-                )
+def upload(
+    import_path: Path | T.Sequence[Path],
+    user_items: config.UserItem,
+    desc_path: str | None = None,
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
+    dry_run=False,
+    skip_subfolders=False,
+) -> None:
+    import_paths = _normalize_import_paths(import_path)
 
-    descs: list[types.DescriptionOrError] = []
+    metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
 
-    if desc_path == "-":
-        try:
-            descs = json.load(sys.stdin)
-        except json.JSONDecodeError as ex:
-            raise exceptions.MapillaryInvalidDescriptionFile(
-                f"Invalid JSON stream from stdin: {ex}"
-            ) from ex
-    else:
-        if not os.path.isfile(desc_path):
-            if is_default_desc_path:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found. Has the directory been processed yet?"
-                )
+    jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
+
+    # Setup the emitter -- the order matters here
+
+    emitter = uploader.EventEmitter()
+
+    # When dry_run mode is on, we disable history by default.
+    # But we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
+    # and when it is on, we enable history regardless of dry_run
+    enable_history = constants.MAPILLARY_UPLOAD_HISTORY_PATH and (
+        not dry_run or constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
+    )
+
+    # Put it first one to check duplications first
+    if enable_history:
+        upload_run_params: JSONDict = {
+            # Null if multiple paths provided
+            "import_path": str(import_path) if isinstance(import_path, Path) else None,
+            "organization_key": user_items.get("MAPOrganizationKey"),
+            "user_key": user_items.get("MAPSettingsUserKey"),
+            "version": VERSION,
+            "run_at": time.time(),
+        }
+        _setup_history(emitter, upload_run_params, metadatas)
+
+    # Set up tdqm
+    _setup_tdqm(emitter)
+
+    # Now stats is empty but it will collect during ALL uploads
+    stats = _setup_api_stats(emitter)
+
+    # Send the progress via IPC, and log the progress in debug mode
+    _setup_ipc(emitter)
+
+    mly_uploader = uploader.Uploader(user_items, emitter=emitter, dry_run=dry_run)
+
+    results = _gen_upload_everything(
+        mly_uploader, metadatas, import_paths, skip_subfolders
+    )
+
+    upload_successes = 0
+    upload_errors: list[Exception] = []
+
+    # The real upload happens sequentially here
+    try:
+        for _, result in results:
+            if result.error is not None:
+                upload_errors.append(_continue_or_fail(result.error))
             else:
-                raise exceptions.MapillaryFileNotFoundError(
-                    f"Description file {desc_path} not found"
-                )
-        with open(desc_path) as fp:
-            try:
-                descs = json.load(fp)
-            except json.JSONDecodeError as ex:
-                raise exceptions.MapillaryInvalidDescriptionFile(
-                    f"Invalid JSON file {desc_path}: {ex}"
-                ) from ex
+                upload_successes += 1
 
-    # the descs load from stdin or json file may contain invalid entries
-    validated_descs = [
-        types.validate_and_fail_desc(desc)
-        for desc in descs
-        # skip error descriptions
-        if "error" not in desc
-    ]
+    except Exception as ex:
+        # Fatal error: log and raise
+        if not dry_run:
+            _api_logging_failed(_summarize(stats), ex)
+        raise ex
 
-    # throw if we found any invalid descs
-    invalid_descs = [desc for desc in validated_descs if "error" in desc]
-    if invalid_descs:
-        for desc in invalid_descs:
-            LOG.error("Invalid description entry: %s", json.dumps(desc))
-        raise exceptions.MapillaryInvalidDescriptionFile(
-            f"Found {len(invalid_descs)} invalid descriptions"
-        )
+    else:
+        if not dry_run:
+            _api_logging_finished(_summarize(stats))
 
-    # validated_descs should contain no errors
-    return [
-        types.from_desc(T.cast(types.Description, desc)) for desc in validated_descs
-    ]
+    finally:
+        # We collected stats after every upload is finished
+        assert upload_successes == len(stats), (
+            f"Expect {upload_successes} success but got {stats}"
+        )
+        _show_upload_summary(stats, upload_errors)
 
 
-def zip_images(
-    import_path: Path,
-    zip_dir: Path,
-    desc_path: str | None = None,
-):
+def zip_images(import_path: Path, zip_dir: Path, desc_path: str | None = None):
     if not import_path.is_dir():
         raise exceptions.MapillaryFileNotFoundError(
             f"Import directory not found: {import_path}"
         )
 
-    metadatas = _load_validate_metadatas_from_desc_path(desc_path, [import_path])
+    metadatas = _load_valid_metadatas_from_desc_path([import_path], desc_path)
 
     if not metadatas:
         LOG.warning("No images or videos found in %s", desc_path)
@@ -141,7 +149,7 @@ def _setup_history(
 ) -> None:
     @emitter.on("upload_start")
     def check_duplication(payload: uploader.Progress):
-        md5sum = payload.get("md5sum")
+        md5sum = payload.get("sequence_md5sum")
         assert md5sum is not None, f"md5sum has to be set for {payload}"
 
         if history.is_uploaded(md5sum):
@@ -164,7 +172,7 @@ def _setup_history(
     @emitter.on("upload_finished")
     def write_history(payload: uploader.Progress):
         sequence_uuid = payload.get("sequence_uuid")
-        md5sum = payload.get("md5sum")
+        md5sum = payload.get("sequence_md5sum")
         assert md5sum is not None, f"md5sum has to be set for {payload}"
 
         if sequence_uuid is None:
@@ -192,8 +200,9 @@ def _setup_history(
 def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
     upload_pbar: tqdm | None = None
 
+    @emitter.on("upload_start")
     @emitter.on("upload_fetch_offset")
-    def upload_fetch_offset(payload: uploader.Progress) -> None:
+    def upload_start(payload: uploader.Progress) -> None:
         nonlocal upload_pbar
 
         if upload_pbar is not None:
@@ -204,18 +213,18 @@ def _setup_tdqm(emitter: uploader.EventEmitter) -> None:
         import_path: str | None = payload.get("import_path")
         filetype = payload.get("file_type", "unknown").upper()
         if import_path is None:
-            _desc = f"Uploading {filetype} ({nth}/{total})"
+            desc = f"Uploading {filetype} ({nth}/{total})"
         else:
-            _desc = (
+            desc = (
                 f"Uploading {filetype} {os.path.basename(import_path)} ({nth}/{total})"
             )
         upload_pbar = tqdm(
             total=payload["entity_size"],
-            desc=_desc,
+            desc=desc,
             unit="B",
             unit_scale=True,
             unit_divisor=1024,
-            initial=payload["offset"],
+            initial=payload.get("offset", 0),
             disable=LOG.getEffectiveLevel() <= logging.DEBUG,
         )
 
@@ -295,8 +304,13 @@ def _setup_api_stats(emitter: uploader.EventEmitter):
 
     @emitter.on("upload_start")
     def collect_start_time(payload: _APIStats) -> None:
-        payload["upload_start_time"] = time.time()
+        now = time.time()
+        payload["upload_start_time"] = now
         payload["upload_total_time"] = 0
+        # These filed should be initialized in upload events like "upload_fetch_offset"
+        # but since we disabled them for uploading images, so we initialize them here
+        payload["upload_last_restart_time"] = now
+        payload["upload_first_offset"] = 0
 
     @emitter.on("upload_fetch_offset")
     def collect_restart_time(payload: _APIStats) -> None:
@@ -417,34 +431,6 @@ def _api_logging_failed(payload: dict, exc: Exception):
         LOG.warning("Error from API Logging for action %s", action, exc_info=True)
 
 
-def _load_descs(
-    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
-    desc_path: str | None,
-    import_paths: T.Sequence[Path],
-) -> list[types.Metadata]:
-    metadatas: list[types.Metadata]
-
-    if _metadatas_from_process is not None:
-        metadatas, _ = types.separate_errors(_metadatas_from_process)
-    else:
-        metadatas = _load_validate_metadatas_from_desc_path(desc_path, import_paths)
-
-    # Make sure all metadatas have sequence uuid assigned
-    # It is used to find the right sequence when writing upload history
-    missing_sequence_uuid = str(uuid.uuid4())
-    for metadata in metadatas:
-        if isinstance(metadata, types.ImageMetadata):
-            if metadata.MAPSequenceUUID is None:
-                metadata.MAPSequenceUUID = missing_sequence_uuid
-
-    for metadata in metadatas:
-        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
-        if isinstance(metadata, types.ImageMetadata):
-            assert metadata.MAPSequenceUUID is not None
-
-    return metadatas
-
-
 _M = T.TypeVar("_M", bound=types.Metadata)
 
 
@@ -466,9 +452,9 @@ def _gen_upload_everything(
         (m for m in metadatas if isinstance(m, types.ImageMetadata)),
         utils.find_images(import_paths, skip_subfolders=skip_subfolders),
     )
-    for image_result in uploader.ZipImageSequence.prepare_images_and_upload(
-        image_metadatas,
+    for image_result in uploader.ZipImageSequence.upload_images(
         mly_uploader,
+        image_metadatas,
     ):
         yield image_result
 
@@ -509,13 +495,9 @@ def _gen_upload_videos(
             "sequence_idx": idx,
             "file_type": video_metadata.filetype.value,
             "import_path": str(video_metadata.filename),
-            "md5sum": video_metadata.md5sum,
+            "sequence_md5sum": video_metadata.md5sum,
         }
 
-        session_key = uploader._session_key(
-            video_metadata.md5sum, upload_api_v4.ClusterFileType.CAMM
-        )
-
         try:
             with video_metadata.filename.open("rb") as src_fp:
                 # Build the mp4 stream with the CAMM samples
@@ -524,12 +506,15 @@ def _gen_upload_videos(
                 )
 
                 # Upload the mp4 stream
-                cluster_id = mly_uploader.upload_stream(
+                file_handle = mly_uploader.upload_stream(
                     T.cast(T.IO[bytes], camm_fp),
-                    upload_api_v4.ClusterFileType.CAMM,
-                    session_key,
                     progress=T.cast(T.Dict[str, T.Any], progress),
                 )
+            cluster_id = mly_uploader.finish_upload(
+                file_handle,
+                api_v4.ClusterFileType.CAMM,
+                progress=T.cast(T.Dict[str, T.Any], progress),
+            )
         except Exception as ex:
             yield video_metadata, uploader.UploadResult(error=ex)
         else:
@@ -630,100 +615,94 @@ def _continue_or_fail(ex: Exception) -> Exception:
     raise ex
 
 
-def upload(
-    import_path: Path | T.Sequence[Path],
-    user_items: types.UserItem,
-    desc_path: str | None = None,
-    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
-    dry_run=False,
-    skip_subfolders=False,
-) -> None:
-    import_paths = _normalize_import_paths(import_path)
-
-    metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths)
+def _gen_upload_zipfiles(
+    mly_uploader: uploader.Uploader, zip_paths: T.Sequence[Path]
+) -> T.Generator[tuple[Path, uploader.UploadResult], None, None]:
+    for idx, zip_path in enumerate(zip_paths):
+        progress: uploader.SequenceProgress = {
+            "total_sequence_count": len(zip_paths),
+            "sequence_idx": idx,
+            "import_path": str(zip_path),
+            "file_type": types.FileType.ZIP.value,
+            "sequence_md5sum": "",  # Placeholder, will be set in upload_zipfile
+        }
+        try:
+            cluster_id = uploader.ZipImageSequence.upload_zipfile(
+                mly_uploader, zip_path, progress=T.cast(T.Dict[str, T.Any], progress)
+            )
+        except Exception as ex:
+            yield zip_path, uploader.UploadResult(error=ex)
+        else:
+            yield zip_path, uploader.UploadResult(result=cluster_id)
 
-    jsonschema.validate(instance=user_items, schema=types.UserItemSchema)
 
-    # Setup the emitter -- the order matters here
+def _load_descs(
+    _metadatas_from_process: T.Sequence[types.MetadataOrError] | None,
+    import_paths: T.Sequence[Path],
+    desc_path: str | None,
+) -> list[types.Metadata]:
+    metadatas: list[types.Metadata]
 
-    emitter = uploader.EventEmitter()
+    if _metadatas_from_process is not None:
+        metadatas, _ = types.separate_errors(_metadatas_from_process)
+    else:
+        metadatas = _load_valid_metadatas_from_desc_path(import_paths, desc_path)
 
-    # When dry_run mode is on, we disable history by default.
-    # But we need dry_run for tests, so we added MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
-    # and when it is on, we enable history regardless of dry_run
-    enable_history = constants.MAPILLARY_UPLOAD_HISTORY_PATH and (
-        not dry_run or constants.MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN
-    )
+    # Make sure all metadatas have sequence uuid assigned
+    # It is used to find the right sequence when writing upload history
+    missing_sequence_uuid = str(uuid.uuid4())
+    for metadata in metadatas:
+        if isinstance(metadata, types.ImageMetadata):
+            if metadata.MAPSequenceUUID is None:
+                metadata.MAPSequenceUUID = missing_sequence_uuid
 
-    # Put it first one to check duplications first
-    if enable_history:
-        upload_run_params: JSONDict = {
-            # Null if multiple paths provided
-            "import_path": str(import_path) if isinstance(import_path, Path) else None,
-            "organization_key": user_items.get("MAPOrganizationKey"),
-            "user_key": user_items.get("MAPSettingsUserKey"),
-            "version": VERSION,
-            "run_at": time.time(),
-        }
-        _setup_history(emitter, upload_run_params, metadatas)
+    for metadata in metadatas:
+        assert isinstance(metadata, (types.ImageMetadata, types.VideoMetadata))
+        if isinstance(metadata, types.ImageMetadata):
+            assert metadata.MAPSequenceUUID is not None
 
-    # Set up tdqm
-    _setup_tdqm(emitter)
+    return metadatas
 
-    # Now stats is empty but it will collect during ALL uploads
-    stats = _setup_api_stats(emitter)
 
-    # Send the progress via IPC, and log the progress in debug mode
-    _setup_ipc(emitter)
+def _load_valid_metadatas_from_desc_path(
+    import_paths: T.Sequence[Path], desc_path: str | None
+) -> list[types.Metadata]:
+    if desc_path is None:
+        desc_path = _find_desc_path(import_paths)
 
-    mly_uploader = uploader.Uploader(user_items, emitter=emitter, dry_run=dry_run)
+    if desc_path == "-":
+        try:
+            metadatas = DescriptionJSONSerializer.deserialize_stream(sys.stdin.buffer)
+        except json.JSONDecodeError as ex:
+            raise exceptions.MapillaryInvalidDescriptionFile(
+                f"Invalid JSON stream from {desc_path}: {ex}"
+            ) from ex
 
-    results = _gen_upload_everything(
-        mly_uploader, metadatas, import_paths, skip_subfolders
-    )
+    else:
+        if not os.path.isfile(desc_path):
+            raise exceptions.MapillaryFileNotFoundError(
+                f"Description file not found: {desc_path}"
+            )
+        with open(desc_path, "rb") as fp:
+            try:
+                metadatas = DescriptionJSONSerializer.deserialize_stream(fp)
+            except json.JSONDecodeError as ex:
+                raise exceptions.MapillaryInvalidDescriptionFile(
+                    f"Invalid JSON stream from {desc_path}: {ex}"
+                ) from ex
 
-    upload_successes = 0
-    upload_errors: list[Exception] = []
+    return metadatas
 
-    # The real upload happens sequentially here
-    try:
-        for _, result in results:
-            if result.error is not None:
-                upload_errors.append(_continue_or_fail(result.error))
-            else:
-                upload_successes += 1
 
-    except Exception as ex:
-        # Fatal error: log and raise
-        if not dry_run:
-            _api_logging_failed(_summarize(stats), ex)
-        raise ex
+def _find_desc_path(import_paths: T.Sequence[Path]) -> str:
+    if len(import_paths) == 1 and import_paths[0].is_dir():
+        return str(import_paths[0].joinpath(constants.IMAGE_DESCRIPTION_FILENAME))
 
+    if 1 < len(import_paths):
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading multiple paths",
+        )
     else:
-        if not dry_run:
-            _api_logging_finished(_summarize(stats))
-
-    finally:
-        # We collected stats after every upload is finished
-        assert upload_successes == len(stats)
-        _show_upload_summary(stats, upload_errors)
-
-
-def _gen_upload_zipfiles(
-    mly_uploader: uploader.Uploader,
-    zip_paths: T.Sequence[Path],
-) -> T.Generator[tuple[Path, uploader.UploadResult], None, None]:
-    for idx, zip_path in enumerate(zip_paths):
-        progress: uploader.SequenceProgress = {
-            "total_sequence_count": len(zip_paths),
-            "sequence_idx": idx,
-            "import_path": str(zip_path),
-        }
-        try:
-            cluster_id = uploader.ZipImageSequence.prepare_zipfile_and_upload(
-                zip_path, mly_uploader, progress=T.cast(T.Dict[str, T.Any], progress)
-            )
-        except Exception as ex:
-            yield zip_path, uploader.UploadResult(error=ex)
-        else:
-            yield zip_path, uploader.UploadResult(result=cluster_id)
+        raise exceptions.MapillaryBadParameterError(
+            "The description path must be specified (with --desc_path) when uploading a single file",
+        )