cognite-toolkit 0.7.48__tar.gz → 0.7.49__tar.gz

{cognite_toolkit-0.7.48 → cognite_toolkit-0.7.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.48
+Version: 0.7.49
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.48 → cognite_toolkit-0.7.49}/cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -1,13 +1,19 @@
-from collections.abc import Callable, Iterable, Sequence
-from enum import Enum
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from typing import get_args
 
 from rich import print
 from rich.console import Console
 from rich.table import Table
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, HTTPMessage, ItemMessage, SuccessResponseItems
+from cognite_toolkit._cdf_tk.client.http_client import (
+    FailedRequestItems,
+    FailedResponseItems,
+    HTTPClient,
+    SuccessResponseItems,
+)
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import MigrationCreator
 from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import DataMapper
@@ -22,25 +28,30 @@ from cognite_toolkit._cdf_tk.exceptions import (
 )
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.storageio.logger import FileDataLogger, OperationStatus
 from cognite_toolkit._cdf_tk.utils import humanize_collection, safe_write, sanitize_filename
 from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
-from cognite_toolkit._cdf_tk.utils.fileio import Chunk, CSVWriter, NDJsonWriter, SchemaColumn, Uncompressed
+from cognite_toolkit._cdf_tk.utils.fileio import NDJsonWriter, Uncompressed
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
-from cognite_toolkit._cdf_tk.utils.progress_tracker import AVAILABLE_STATUS, ProgressTracker, Status
 
 from .data_model import INSTANCE_SOURCE_VIEW_ID, MODEL_ID, RESOURCE_VIEW_MAPPING_VIEW_ID
+from .issues import WriteIssue
 
 
-class MigrationCommand(ToolkitCommand):
-    class Steps(str, Enum):
-        DOWNLOAD = "download"
-        CONVERT = "convert"
-        UPLOAD = "upload"
+@dataclass
+class OperationIssue:
+    message: str
+    count: int
+
 
-        @classmethod
-        def list(cls) -> list[str]:
-            return [step.value for step in cls.__members__.values()]
+@dataclass
+class MigrationStatusResult:
+    status: OperationStatus
+    issues: list[OperationIssue]
+    count: int
 
+
+class MigrationCommand(ToolkitCommand):
     def migrate(
         self,
         selected: T_Selector,
@@ -49,7 +60,7 @@ class MigrationCommand(ToolkitCommand):
         log_dir: Path,
         dry_run: bool = False,
         verbose: bool = False,
-    ) -> ProgressTracker[str]:
+    ) -> list[MigrationStatusResult]:
         if log_dir.exists() and any(log_dir.iterdir()):
             raise ToolkitFileExistsError(
                 f"Log directory {log_dir} already exists. Please remove it or choose another directory."
@@ -65,15 +76,18 @@ class MigrationCommand(ToolkitCommand):
             self.validate_available_capacity(data.client, total_items)
 
         console = Console()
-        tracker = ProgressTracker[str](self.Steps.list())
         with (
             NDJsonWriter(log_dir, kind=f"{selected.kind}MigrationIssues", compression=Uncompressed) as log_file,
             HTTPClient(config=data.client.config) as write_client,
         ):
+            logger = FileDataLogger(log_file)
+            data.logger = logger
+            mapper.logger = logger
+
             executor = ProducerWorkerExecutor[Sequence[T_ResourceResponse], Sequence[UploadItem[T_ResourceRequest]]](
-                download_iterable=self._download_iterable(selected, data, tracker),
-                process=self._convert(mapper, data, tracker, log_file),
-                write=self._upload(selected, write_client, data, tracker, log_file, dry_run),
+                download_iterable=(page.items for page in data.stream_data(selected)),
+                process=self._convert(mapper, data),
+                write=self._upload(selected, write_client, data, dry_run),
                 iteration_count=iteration_count,
                 max_queue_size=10,
                 download_description=f"Downloading {selected.display_name}",
@@ -86,91 +100,71 @@ class MigrationCommand(ToolkitCommand):
             executor.run()
             total = executor.total_items
 
-        self._print_table(tracker.aggregate(), console)
-        self._print_csv(tracker, log_dir, f"{selected.kind}Items", console)
+        results = self._create_status_summary(logger)
+
+        self._print_rich_tables(results, console)
+        self._print_txt(results, log_dir, f"{selected.kind}Items", console)
         executor.raise_on_error()
         action = "Would migrate" if dry_run else "Migrating"
         console.print(f"{action} {total:,} {selected.display_name} to instances.")
-        return tracker
 
-    def _print_table(self, results: dict[tuple[str, Status], int], console: Console) -> None:
-        for step in self.Steps:
-            # We treat pending as failed for summary purposes
-            results[(step.value, "failed")] = results.get((step.value, "failed"), 0) + results.get(
-                (step.value, "pending"), 0
+        return results
+
+    # Todo: Move to the logger module
+    @classmethod
+    def _create_status_summary(cls, logger: FileDataLogger) -> list[MigrationStatusResult]:
+        results: list[MigrationStatusResult] = []
+        status_counts = logger.tracker.get_status_counts()
+        for status in get_args(OperationStatus):
+            issue_counts = logger.tracker.get_issue_counts(status)
+            issues = [OperationIssue(message=issue, count=count) for issue, count in issue_counts.items()]
+            result = MigrationStatusResult(
+                status=status,
+                issues=issues,
+                count=status_counts.get(status, 0),
             )
+            results.append(result)
+        return results
 
+    def _print_rich_tables(self, results: list[MigrationStatusResult], console: Console) -> None:
         table = Table(title="Migration Summary", show_lines=True)
-        table.add_column("Status", style="cyan", no_wrap=True)
-        for step in self.Steps:
-            table.add_column(step.value.capitalize(), style="magenta")
-        for status in AVAILABLE_STATUS:
-            if status == "pending":
-                # Skip pending as we treat it as failed
-                continue
-            row = [status]
-            for step in self.Steps:
-                row.append(str(results.get((step.value, status), 0)))
-            table.add_row(*row)
-
+        table.add_column("Status", style="bold")
+        table.add_column("Count", justify="right", style="bold")
+        table.add_column("Issues", style="bold")
+        for result in results:
+            issues_str = "\n".join(f"{issue.message}: {issue.count}" for issue in result.issues) or ""
+            table.add_row(result.status, str(result.count), issues_str)
         console.print(table)
 
-    def _print_csv(self, tracker: ProgressTracker[str], log_dir: Path, kind: str, console: Console) -> None:
-        with CSVWriter(log_dir, kind=kind, compression=Uncompressed, columns=self._csv_columns()) as csv_file:
-            batch: list[Chunk] = []
-            steps = self.Steps.list()
-            for item_id, progress in tracker.result().items():
-                batch.append({"ID": str(item_id), **{step: progress[step] for step in steps}})
-                if len(batch) >= 1000:
-                    csv_file.write_chunks(batch)
-                    batch = []
-            if batch:
-                csv_file.write_chunks(batch)
-        console.print(f"Migration items written to {log_dir}")
-
-    @classmethod
-    def _csv_columns(cls) -> list[SchemaColumn]:
-        return [
-            SchemaColumn(name="ID", type="string"),
-            *(SchemaColumn(name=step, type="string") for step in cls.Steps.list()),
-        ]
-
-    def _download_iterable(
-        self,
-        selected: T_Selector,
-        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-    ) -> Iterable[Sequence[T_ResourceResponse]]:
-        for page in data.stream_data(selected):
-            for item in page.items:
-                tracker.set_progress(data.as_id(item), self.Steps.DOWNLOAD, "success")
-            yield page.items
+    def _print_txt(self, results: list[MigrationStatusResult], log_dir: Path, kind: str, console: Console) -> None:
+        summary_file = log_dir / f"{kind}_migration_summary.txt"
+        with summary_file.open("w", encoding="utf-8") as f:
+            f.write("Migration Summary\n")
+            f.write("=================\n\n")
+            for result in results:
+                f.write(f"Status: {result.status}\n")
+                f.write(f"Count: {result.count}\n")
+                f.write("Issues:\n")
+                if result.issues:
+                    for issue in result.issues:
+                        f.write(f"  - {issue.message}: {issue.count}\n")
+                else:
+                    f.write("  None\n")
+                f.write("\n")
+        console.print(f"Summary written to {log_dir}")
 
+    @staticmethod
     def _convert(
-        self,
         mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
         data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-        log_file: NDJsonWriter,
     ) -> Callable[[Sequence[T_ResourceResponse]], Sequence[UploadItem[T_ResourceRequest]]]:
         def track_mapping(source: Sequence[T_ResourceResponse]) -> list[UploadItem[T_ResourceRequest]]:
             mapped = mapper.map(source)
-            issues: list[Chunk] = []
-            targets: list[UploadItem[T_ResourceRequest]] = []
-
-            for (target, issue), item in zip(mapped, source):
-                id_ = data.as_id(item)
-                result: Status = "failed" if target is None else "success"
-                tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)
-
-                if issue.has_issues:
-                    # MyPy fails to understand that dict[str, JsonVal] is a Chunk
-                    issues.append(issue.dump())  # type: ignore[arg-type]
-                if target is not None:
-                    targets.append(UploadItem(source_id=id_, item=target))
-            if issues:
-                log_file.write_chunks(issues)
-            return targets
+            return [
+                UploadItem(source_id=data.as_id(item), item=target)
+                for target, item in zip(mapped, source)
+                if target is not None
+            ]
 
         return track_mapping
 
@@ -179,36 +173,37 @@ class MigrationCommand(ToolkitCommand):
         selected: T_Selector,
         write_client: HTTPClient,
         target: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-        log_file: NDJsonWriter,
         dry_run: bool,
     ) -> Callable[[Sequence[UploadItem[T_ResourceRequest]]], None]:
         def upload_items(data_item: Sequence[UploadItem[T_ResourceRequest]]) -> None:
             if not data_item:
                 return None
-            responses: Sequence[HTTPMessage]
             if dry_run:
-                responses = [
-                    SuccessResponseItems(
-                        status_code=200, body="", content=b"", ids=[item.source_id for item in data_item]
-                    )
-                ]
-            else:
-                responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
-
-            issues: list[Chunk] = []
+                target.logger.tracker.finalize_item([item.source_id for item in data_item], "pending")
+                return None
+
+            responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
+
+            # Todo: Move logging into the UploadableStorageIO class
+            issues: list[WriteIssue] = []
             for item in responses:
                 if isinstance(item, SuccessResponseItems):
-                    for success_id in item.ids:
-                        tracker.set_progress(success_id, step=self.Steps.UPLOAD, status="success")
-                elif isinstance(item, ItemMessage):
-                    for failed_id in item.ids:
-                        tracker.set_progress(failed_id, step=self.Steps.UPLOAD, status="failed")
-
-                if not isinstance(item, SuccessResponseItems):
-                    issues.append(item.dump())  # type: ignore[arg-type]
+                    target.logger.tracker.finalize_item(item.ids, "success")
+                    continue
+                if isinstance(item, FailedResponseItems):
+                    error = item.error
+                    for id_ in item.ids:
+                        issue = WriteIssue(id=str(id_), status_code=error.code, message=error.message)
+                        issues.append(issue)
+                elif isinstance(item, FailedRequestItems):
+                    for id_ in item.ids:
+                        issue = WriteIssue(id=str(id_), status_code=0, message=item.error)
+                        issues.append(issue)
+
+                if isinstance(item, FailedResponseItems | FailedRequestItems):
+                    target.logger.tracker.finalize_item(item.ids, "failure")
             if issues:
-                log_file.write_chunks(issues)
+                target.logger.log(issues)
             return None
 
         return upload_items

{cognite_toolkit-0.7.48 → cognite_toolkit-0.7.49}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -17,6 +17,7 @@ from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.resource_classes.asset import AssetResponse
+from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
 from cognite_toolkit._cdf_tk.client.resource_classes.event import EventResponse
 from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataResponse
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import (
@@ -198,7 +199,11 @@ def asset_centric_to_dm(
     data_set_id = dumped.pop("dataSetId", None)
     external_id = dumped.pop("externalId", None)
 
-    issue = ConversionIssue(asset_centric_id=AssetCentricId(resource_type, id_=id_), instance_id=instance_id)
+    issue = ConversionIssue(
+        id=str(AssetCentricId(resource_type, id_=id_)),
+        asset_centric_id=AssetCentricId(resource_type, id_=id_),
+        instance_id=NodeReference(space=instance_id.space, external_id=instance_id.external_id),
+    )
 
     properties = create_properties(
         dumped,

{cognite_toolkit-0.7.48 → cognite_toolkit-0.7.49}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -46,13 +46,13 @@ from cognite_toolkit._cdf_tk.commands._migrate.issues import (
     CanvasMigrationIssue,
     ChartMigrationIssue,
     ConversionIssue,
-    MigrationIssue,
     ThreeDModelMigrationIssue,
 )
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector
+from cognite_toolkit._cdf_tk.storageio.logger import DataLogger, NoOpLogger
 from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector, ThreeDSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types2 import T_AssetCentricResourceExtended
@@ -62,6 +62,10 @@ from .selectors import AssetCentricMigrationSelector
 
 
 class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
+    def __init__(self, client: ToolkitClient) -> None:
+        self.client = client
+        self.logger: DataLogger = NoOpLogger()
+
     def prepare(self, source_selector: T_Selector) -> None:
         """Prepare the data mapper with the given source selector.
 
@@ -73,14 +77,14 @@ class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC
         pass
 
     @abstractmethod
-    def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[tuple[T_ResourceRequest | None, MigrationIssue]]:
+    def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[T_ResourceRequest | None]:
         """Map a chunk of source data to the target format.
 
         Args:
             source: The source data chunk to be mapped.
 
         Returns:
-            A tuple containing the mapped data and a list of any issues encountered during mapping.
+            A sequence of mapped target data.
 
         """
         raise NotImplementedError("Subclasses must implement this method.")
@@ -90,7 +94,7 @@ class AssetCentricMapper(
     DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResourceExtended], InstanceApply]
 ):
     def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
+        super().__init__(client)
         self._ingestion_view_by_id: dict[ViewId, View] = {}
         self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
         self._direct_relation_cache = DirectRelationCache(client)
@@ -118,14 +122,37 @@ class AssetCentricMapper(
 
     def map(
         self, source: Sequence[AssetCentricMapping[T_AssetCentricResourceExtended]]
-    ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
+    ) -> Sequence[InstanceApply | None]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
         # We update the direct relation cache in bulk for all resources in the chunk.
         self._direct_relation_cache.update(item.resource for item in source)
-        output: list[tuple[InstanceApply | None, ConversionIssue]] = []
+        output: list[InstanceApply | None] = []
+        issues: list[ConversionIssue] = []
         for item in source:
             instance, conversion_issue = self._map_single_item(item)
-            output.append((instance, conversion_issue))
+            identifier = str(item.mapping.as_asset_centric_id())
+
+            if conversion_issue.missing_instance_space:
+                self.logger.tracker.add_issue(identifier, "Missing instance space")
+            if conversion_issue.failed_conversions:
+                self.logger.tracker.add_issue(identifier, "Failed conversions")
+            if conversion_issue.invalid_instance_property_types:
+                self.logger.tracker.add_issue(identifier, "Invalid instance property types")
+            if conversion_issue.missing_asset_centric_properties:
+                self.logger.tracker.add_issue(identifier, "Missing asset-centric properties")
+            if conversion_issue.missing_instance_properties:
+                self.logger.tracker.add_issue(identifier, "Missing instance properties")
+            if conversion_issue.ignored_asset_centric_properties:
+                self.logger.tracker.add_issue(identifier, "Ignored asset-centric properties")
+
+            if conversion_issue.has_issues:
+                issues.append(conversion_issue)
+
+            if instance is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(instance)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _map_single_item(
@@ -154,15 +181,29 @@ class AssetCentricMapper(
 
 
 class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(self, source: Sequence[Chart]) -> Sequence[tuple[ChartWrite | None, MigrationIssue]]:
+    def map(self, source: Sequence[Chart]) -> Sequence[ChartWrite | None]:
         self._populate_cache(source)
-        output: list[tuple[ChartWrite | None, MigrationIssue]] = []
+        output: list[ChartWrite | None] = []
+        issues: list[ChartMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.external_id
+
+            if issue.missing_timeseries_ids:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries IDs")
+            if issue.missing_timeseries_external_ids:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries external IDs")
+            if issue.missing_timeseries_identifier:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries identifier")
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[Chart]) -> None:
@@ -184,7 +225,7 @@ class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
             self.client.migration.lookup.time_series(external_id=list(timeseries_external_ids))
 
     def _map_single_item(self, item: Chart) -> tuple[ChartWrite | None, ChartMigrationIssue]:
-        issue = ChartMigrationIssue(chart_external_id=item.external_id)
+        issue = ChartMigrationIssue(chart_external_id=item.external_id, id=item.external_id)
         time_series_collection = item.data.time_series_collection or []
         timeseries_core_collection = self._create_timeseries_core_collection(time_series_collection, issue)
         if issue.has_issues:
@@ -280,16 +321,30 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
     DEFAULT_TIMESERIES_VIEW = ViewId("cdf_cdm", "CogniteTimeSeries", "v1")
 
     def __init__(self, client: ToolkitClient, dry_run: bool, skip_on_missing_ref: bool = False) -> None:
-        self.client = client
+        super().__init__(client)
         self.dry_run = dry_run
         self.skip_on_missing_ref = skip_on_missing_ref
 
-    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[tuple[IndustrialCanvasApply | None, MigrationIssue]]:
+    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[IndustrialCanvasApply | None]:
         self._populate_cache(source)
-        output: list[tuple[IndustrialCanvasApply | None, MigrationIssue]] = []
+        output: list[IndustrialCanvasApply | None] = []
+        issues: list[CanvasMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.as_id()
+
+            if issue.missing_reference_ids:
+                self.logger.tracker.add_issue(identifier, "Missing reference IDs")
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     @property
@@ -340,7 +395,9 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
 
     def _map_single_item(self, canvas: IndustrialCanvas) -> tuple[IndustrialCanvasApply | None, CanvasMigrationIssue]:
         update = canvas.as_write()
-        issue = CanvasMigrationIssue(canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name)
+        issue = CanvasMigrationIssue(
+            canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name, id=canvas.canvas.name
+        )
 
         remaining_container_references: list[ContainerReferenceApply] = []
         new_fdm_references: list[FdmInstanceContainerReferenceApply] = []
@@ -399,17 +456,26 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
 
 
 class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(
-        self, source: Sequence[ThreeDModelResponse]
-    ) -> Sequence[tuple[ThreeDMigrationRequest | None, MigrationIssue]]:
+    def map(self, source: Sequence[ThreeDModelResponse]) -> Sequence[ThreeDMigrationRequest | None]:
         self._populate_cache(source)
-        output: list[tuple[ThreeDMigrationRequest | None, MigrationIssue]] = []
+        output: list[ThreeDMigrationRequest | None] = []
+        issues: list[ThreeDModelMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.name
+
+            if issue.error_message:
+                for error in issue.error_message:
+                    self.logger.tracker.add_issue(identifier, error)
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[ThreeDModelResponse]) -> None:
@@ -422,7 +488,7 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
     def _map_single_item(
         self, item: ThreeDModelResponse
     ) -> tuple[ThreeDMigrationRequest | None, ThreeDModelMigrationIssue]:
-        issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id)
+        issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id, id=item.name)
         instance_space: str | None = None
         last_revision_id: int | None = None
         model_type: Literal["CAD", "PointCloud", "Image360"] | None = None
@@ -478,17 +544,27 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
 
 
 class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingClassicResponse, AssetMappingDMRequest]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(
-        self, source: Sequence[AssetMappingClassicResponse]
-    ) -> Sequence[tuple[AssetMappingDMRequest | None, MigrationIssue]]:
-        output: list[tuple[AssetMappingDMRequest | None, MigrationIssue]] = []
+    def map(self, source: Sequence[AssetMappingClassicResponse]) -> Sequence[AssetMappingDMRequest | None]:
+        output: list[AssetMappingDMRequest | None] = []
+        issues: list[ThreeDModelMigrationIssue] = []
         self._populate_cache(source)
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = f"AssetMapping_{item.model_id!s}_{item.revision_id!s}_{item.asset_id!s}"
+
+            if issue.error_message:
+                for error in issue.error_message:
+                    self.logger.tracker.add_issue(identifier, error)
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[AssetMappingClassicResponse]) -> None:
@@ -501,7 +577,9 @@ class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingClassicResponse,
     def _map_single_item(
         self, item: AssetMappingClassicResponse
     ) -> tuple[AssetMappingDMRequest | None, ThreeDModelMigrationIssue]:
-        issue = ThreeDModelMigrationIssue(model_name=f"AssetMapping_{item.model_id}", model_id=item.model_id)
+        issue = ThreeDModelMigrationIssue(
+            model_name=f"AssetMapping_{item.model_id}", model_id=item.model_id, id=f"AssetMapping_{item.model_id}"
+        )
         asset_instance_id = item.asset_instance_id
         if item.asset_id and asset_instance_id is None:
             asset_node_id = self.client.migration.lookup.assets(item.asset_id)