cognite-toolkit 0.7.48__py3-none-any.whl → 0.7.49__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -1,13 +1,19 @@
-from collections.abc import Callable, Iterable, Sequence
-from enum import Enum
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from typing import get_args
 
 from rich import print
 from rich.console import Console
 from rich.table import Table
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, HTTPMessage, ItemMessage, SuccessResponseItems
+from cognite_toolkit._cdf_tk.client.http_client import (
+    FailedRequestItems,
+    FailedResponseItems,
+    HTTPClient,
+    SuccessResponseItems,
+)
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands._migrate.creators import MigrationCreator
 from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import DataMapper
@@ -22,25 +28,30 @@ from cognite_toolkit._cdf_tk.exceptions import (
 )
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.storageio.logger import FileDataLogger, OperationStatus
 from cognite_toolkit._cdf_tk.utils import humanize_collection, safe_write, sanitize_filename
 from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
-from cognite_toolkit._cdf_tk.utils.fileio import Chunk, CSVWriter, NDJsonWriter, SchemaColumn, Uncompressed
+from cognite_toolkit._cdf_tk.utils.fileio import NDJsonWriter, Uncompressed
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
-from cognite_toolkit._cdf_tk.utils.progress_tracker import AVAILABLE_STATUS, ProgressTracker, Status
 
 from .data_model import INSTANCE_SOURCE_VIEW_ID, MODEL_ID, RESOURCE_VIEW_MAPPING_VIEW_ID
+from .issues import WriteIssue
 
 
-class MigrationCommand(ToolkitCommand):
-    class Steps(str, Enum):
-        DOWNLOAD = "download"
-        CONVERT = "convert"
-        UPLOAD = "upload"
+@dataclass
+class OperationIssue:
+    message: str
+    count: int
+
 
-        @classmethod
-        def list(cls) -> list[str]:
-            return [step.value for step in cls.__members__.values()]
+@dataclass
+class MigrationStatusResult:
+    status: OperationStatus
+    issues: list[OperationIssue]
+    count: int
 
+
+class MigrationCommand(ToolkitCommand):
     def migrate(
         self,
         selected: T_Selector,
@@ -49,7 +60,7 @@ class MigrationCommand(ToolkitCommand):
         log_dir: Path,
         dry_run: bool = False,
         verbose: bool = False,
-    ) -> ProgressTracker[str]:
+    ) -> list[MigrationStatusResult]:
         if log_dir.exists() and any(log_dir.iterdir()):
             raise ToolkitFileExistsError(
                 f"Log directory {log_dir} already exists. Please remove it or choose another directory."
@@ -65,15 +76,18 @@ class MigrationCommand(ToolkitCommand):
             self.validate_available_capacity(data.client, total_items)
 
         console = Console()
-        tracker = ProgressTracker[str](self.Steps.list())
         with (
             NDJsonWriter(log_dir, kind=f"{selected.kind}MigrationIssues", compression=Uncompressed) as log_file,
             HTTPClient(config=data.client.config) as write_client,
         ):
+            logger = FileDataLogger(log_file)
+            data.logger = logger
+            mapper.logger = logger
+
             executor = ProducerWorkerExecutor[Sequence[T_ResourceResponse], Sequence[UploadItem[T_ResourceRequest]]](
-                download_iterable=self._download_iterable(selected, data, tracker),
-                process=self._convert(mapper, data, tracker, log_file),
-                write=self._upload(selected, write_client, data, tracker, log_file, dry_run),
+                download_iterable=(page.items for page in data.stream_data(selected)),
+                process=self._convert(mapper, data),
+                write=self._upload(selected, write_client, data, dry_run),
                 iteration_count=iteration_count,
                 max_queue_size=10,
                 download_description=f"Downloading {selected.display_name}",
@@ -86,91 +100,71 @@ class MigrationCommand(ToolkitCommand):
             executor.run()
             total = executor.total_items
 
-        self._print_table(tracker.aggregate(), console)
-        self._print_csv(tracker, log_dir, f"{selected.kind}Items", console)
+        results = self._create_status_summary(logger)
+
+        self._print_rich_tables(results, console)
+        self._print_txt(results, log_dir, f"{selected.kind}Items", console)
         executor.raise_on_error()
         action = "Would migrate" if dry_run else "Migrating"
         console.print(f"{action} {total:,} {selected.display_name} to instances.")
-        return tracker
 
-    def _print_table(self, results: dict[tuple[str, Status], int], console: Console) -> None:
-        for step in self.Steps:
-            # We treat pending as failed for summary purposes
-            results[(step.value, "failed")] = results.get((step.value, "failed"), 0) + results.get(
-                (step.value, "pending"), 0
+        return results
+
+    # Todo: Move to the logger module
+    @classmethod
+    def _create_status_summary(cls, logger: FileDataLogger) -> list[MigrationStatusResult]:
+        results: list[MigrationStatusResult] = []
+        status_counts = logger.tracker.get_status_counts()
+        for status in get_args(OperationStatus):
+            issue_counts = logger.tracker.get_issue_counts(status)
+            issues = [OperationIssue(message=issue, count=count) for issue, count in issue_counts.items()]
+            result = MigrationStatusResult(
+                status=status,
+                issues=issues,
+                count=status_counts.get(status, 0),
             )
+            results.append(result)
+        return results
 
+    def _print_rich_tables(self, results: list[MigrationStatusResult], console: Console) -> None:
         table = Table(title="Migration Summary", show_lines=True)
-        table.add_column("Status", style="cyan", no_wrap=True)
-        for step in self.Steps:
-            table.add_column(step.value.capitalize(), style="magenta")
-        for status in AVAILABLE_STATUS:
-            if status == "pending":
-                # Skip pending as we treat it as failed
-                continue
-            row = [status]
-            for step in self.Steps:
-                row.append(str(results.get((step.value, status), 0)))
-            table.add_row(*row)
-
+        table.add_column("Status", style="bold")
+        table.add_column("Count", justify="right", style="bold")
+        table.add_column("Issues", style="bold")
+        for result in results:
+            issues_str = "\n".join(f"{issue.message}: {issue.count}" for issue in result.issues) or ""
+            table.add_row(result.status, str(result.count), issues_str)
         console.print(table)
 
-    def _print_csv(self, tracker: ProgressTracker[str], log_dir: Path, kind: str, console: Console) -> None:
-        with CSVWriter(log_dir, kind=kind, compression=Uncompressed, columns=self._csv_columns()) as csv_file:
-            batch: list[Chunk] = []
-            steps = self.Steps.list()
-            for item_id, progress in tracker.result().items():
-                batch.append({"ID": str(item_id), **{step: progress[step] for step in steps}})
-                if len(batch) >= 1000:
-                    csv_file.write_chunks(batch)
-                    batch = []
-            if batch:
-                csv_file.write_chunks(batch)
-        console.print(f"Migration items written to {log_dir}")
-
-    @classmethod
-    def _csv_columns(cls) -> list[SchemaColumn]:
-        return [
-            SchemaColumn(name="ID", type="string"),
-            *(SchemaColumn(name=step, type="string") for step in cls.Steps.list()),
-        ]
-
-    def _download_iterable(
-        self,
-        selected: T_Selector,
-        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-    ) -> Iterable[Sequence[T_ResourceResponse]]:
-        for page in data.stream_data(selected):
-            for item in page.items:
-                tracker.set_progress(data.as_id(item), self.Steps.DOWNLOAD, "success")
-            yield page.items
+    def _print_txt(self, results: list[MigrationStatusResult], log_dir: Path, kind: str, console: Console) -> None:
+        summary_file = log_dir / f"{kind}_migration_summary.txt"
+        with summary_file.open("w", encoding="utf-8") as f:
+            f.write("Migration Summary\n")
+            f.write("=================\n\n")
+            for result in results:
+                f.write(f"Status: {result.status}\n")
+                f.write(f"Count: {result.count}\n")
+                f.write("Issues:\n")
+                if result.issues:
+                    for issue in result.issues:
+                        f.write(f"  - {issue.message}: {issue.count}\n")
+                else:
+                    f.write("  None\n")
+                f.write("\n")
+        console.print(f"Summary written to {log_dir}")
 
+    @staticmethod
     def _convert(
-        self,
         mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
         data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-        log_file: NDJsonWriter,
     ) -> Callable[[Sequence[T_ResourceResponse]], Sequence[UploadItem[T_ResourceRequest]]]:
         def track_mapping(source: Sequence[T_ResourceResponse]) -> list[UploadItem[T_ResourceRequest]]:
             mapped = mapper.map(source)
-            issues: list[Chunk] = []
-            targets: list[UploadItem[T_ResourceRequest]] = []
-
-            for (target, issue), item in zip(mapped, source):
-                id_ = data.as_id(item)
-                result: Status = "failed" if target is None else "success"
-                tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)
-
-                if issue.has_issues:
-                    # MyPy fails to understand that dict[str, JsonVal] is a Chunk
-                    issues.append(issue.dump())  # type: ignore[arg-type]
-                if target is not None:
-                    targets.append(UploadItem(source_id=id_, item=target))
-            if issues:
-                log_file.write_chunks(issues)
-            return targets
+            return [
+                UploadItem(source_id=data.as_id(item), item=target)
+                for target, item in zip(mapped, source)
+                if target is not None
+            ]
 
         return track_mapping
 
@@ -179,36 +173,37 @@ class MigrationCommand(ToolkitCommand):
         selected: T_Selector,
         write_client: HTTPClient,
         target: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
-        tracker: ProgressTracker[str],
-        log_file: NDJsonWriter,
         dry_run: bool,
     ) -> Callable[[Sequence[UploadItem[T_ResourceRequest]]], None]:
         def upload_items(data_item: Sequence[UploadItem[T_ResourceRequest]]) -> None:
             if not data_item:
                 return None
-            responses: Sequence[HTTPMessage]
             if dry_run:
-                responses = [
-                    SuccessResponseItems(
-                        status_code=200, body="", content=b"", ids=[item.source_id for item in data_item]
-                    )
-                ]
-            else:
-                responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
-
-            issues: list[Chunk] = []
+                target.logger.tracker.finalize_item([item.source_id for item in data_item], "pending")
+                return None
+
+            responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
+
+            # Todo: Move logging into the UploadableStorageIO class
+            issues: list[WriteIssue] = []
             for item in responses:
                 if isinstance(item, SuccessResponseItems):
-                    for success_id in item.ids:
-                        tracker.set_progress(success_id, step=self.Steps.UPLOAD, status="success")
-                elif isinstance(item, ItemMessage):
-                    for failed_id in item.ids:
-                        tracker.set_progress(failed_id, step=self.Steps.UPLOAD, status="failed")
-
-                if not isinstance(item, SuccessResponseItems):
-                    issues.append(item.dump())  # type: ignore[arg-type]
+                    target.logger.tracker.finalize_item(item.ids, "success")
+                    continue
+                if isinstance(item, FailedResponseItems):
+                    error = item.error
+                    for id_ in item.ids:
+                        issue = WriteIssue(id=str(id_), status_code=error.code, message=error.message)
+                        issues.append(issue)
+                elif isinstance(item, FailedRequestItems):
+                    for id_ in item.ids:
+                        issue = WriteIssue(id=str(id_), status_code=0, message=item.error)
+                        issues.append(issue)
+
+                if isinstance(item, FailedResponseItems | FailedRequestItems):
+                    target.logger.tracker.finalize_item(item.ids, "failure")
             if issues:
-                log_file.write_chunks(issues)
+                target.logger.log(issues)
             return None
 
         return upload_items

cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -17,6 +17,7 @@ from cognite.client.utils._identifier import InstanceId
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.resource_classes.asset import AssetResponse
+from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
 from cognite_toolkit._cdf_tk.client.resource_classes.event import EventResponse
 from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataResponse
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import (
@@ -198,7 +199,11 @@ def asset_centric_to_dm(
     data_set_id = dumped.pop("dataSetId", None)
     external_id = dumped.pop("externalId", None)
 
-    issue = ConversionIssue(asset_centric_id=AssetCentricId(resource_type, id_=id_), instance_id=instance_id)
+    issue = ConversionIssue(
+        id=str(AssetCentricId(resource_type, id_=id_)),
+        asset_centric_id=AssetCentricId(resource_type, id_=id_),
+        instance_id=NodeReference(space=instance_id.space, external_id=instance_id.external_id),
+    )
 
     properties = create_properties(
         dumped,

cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -46,13 +46,13 @@ from cognite_toolkit._cdf_tk.commands._migrate.issues import (
     CanvasMigrationIssue,
     ChartMigrationIssue,
     ConversionIssue,
-    MigrationIssue,
     ThreeDModelMigrationIssue,
 )
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector
+from cognite_toolkit._cdf_tk.storageio.logger import DataLogger, NoOpLogger
 from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector, ThreeDSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types2 import T_AssetCentricResourceExtended
@@ -62,6 +62,10 @@ from .selectors import AssetCentricMigrationSelector
 
 
 class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
+    def __init__(self, client: ToolkitClient) -> None:
+        self.client = client
+        self.logger: DataLogger = NoOpLogger()
+
     def prepare(self, source_selector: T_Selector) -> None:
         """Prepare the data mapper with the given source selector.
 
@@ -73,14 +77,14 @@ class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC
         pass
 
     @abstractmethod
-    def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[tuple[T_ResourceRequest | None, MigrationIssue]]:
+    def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[T_ResourceRequest | None]:
         """Map a chunk of source data to the target format.
 
         Args:
             source: The source data chunk to be mapped.
 
         Returns:
-            A tuple containing the mapped data and a list of any issues encountered during mapping.
+            A sequence of mapped target data.
 
         """
         raise NotImplementedError("Subclasses must implement this method.")
@@ -90,7 +94,7 @@ class AssetCentricMapper(
     DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResourceExtended], InstanceApply]
 ):
     def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
+        super().__init__(client)
         self._ingestion_view_by_id: dict[ViewId, View] = {}
         self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
         self._direct_relation_cache = DirectRelationCache(client)
@@ -118,14 +122,37 @@ class AssetCentricMapper(
 
     def map(
         self, source: Sequence[AssetCentricMapping[T_AssetCentricResourceExtended]]
-    ) -> Sequence[tuple[InstanceApply | None, ConversionIssue]]:
+    ) -> Sequence[InstanceApply | None]:
         """Map a chunk of asset-centric data to InstanceApplyList format."""
         # We update the direct relation cache in bulk for all resources in the chunk.
         self._direct_relation_cache.update(item.resource for item in source)
-        output: list[tuple[InstanceApply | None, ConversionIssue]] = []
+        output: list[InstanceApply | None] = []
+        issues: list[ConversionIssue] = []
         for item in source:
             instance, conversion_issue = self._map_single_item(item)
-            output.append((instance, conversion_issue))
+            identifier = str(item.mapping.as_asset_centric_id())
+
+            if conversion_issue.missing_instance_space:
+                self.logger.tracker.add_issue(identifier, "Missing instance space")
+            if conversion_issue.failed_conversions:
+                self.logger.tracker.add_issue(identifier, "Failed conversions")
+            if conversion_issue.invalid_instance_property_types:
+                self.logger.tracker.add_issue(identifier, "Invalid instance property types")
+            if conversion_issue.missing_asset_centric_properties:
+                self.logger.tracker.add_issue(identifier, "Missing asset-centric properties")
+            if conversion_issue.missing_instance_properties:
+                self.logger.tracker.add_issue(identifier, "Missing instance properties")
+            if conversion_issue.ignored_asset_centric_properties:
+                self.logger.tracker.add_issue(identifier, "Ignored asset-centric properties")
+
+            if conversion_issue.has_issues:
+                issues.append(conversion_issue)
+
+            if instance is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(instance)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _map_single_item(
@@ -154,15 +181,29 @@ class AssetCentricMapper(
 
 
 class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(self, source: Sequence[Chart]) -> Sequence[tuple[ChartWrite | None, MigrationIssue]]:
+    def map(self, source: Sequence[Chart]) -> Sequence[ChartWrite | None]:
         self._populate_cache(source)
-        output: list[tuple[ChartWrite | None, MigrationIssue]] = []
+        output: list[ChartWrite | None] = []
+        issues: list[ChartMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.external_id
+
+            if issue.missing_timeseries_ids:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries IDs")
+            if issue.missing_timeseries_external_ids:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries external IDs")
+            if issue.missing_timeseries_identifier:
+                self.logger.tracker.add_issue(identifier, "Missing timeseries identifier")
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[Chart]) -> None:
@@ -184,7 +225,7 @@ class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
             self.client.migration.lookup.time_series(external_id=list(timeseries_external_ids))
 
     def _map_single_item(self, item: Chart) -> tuple[ChartWrite | None, ChartMigrationIssue]:
-        issue = ChartMigrationIssue(chart_external_id=item.external_id)
+        issue = ChartMigrationIssue(chart_external_id=item.external_id, id=item.external_id)
         time_series_collection = item.data.time_series_collection or []
         timeseries_core_collection = self._create_timeseries_core_collection(time_series_collection, issue)
         if issue.has_issues:
@@ -280,16 +321,30 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
     DEFAULT_TIMESERIES_VIEW = ViewId("cdf_cdm", "CogniteTimeSeries", "v1")
 
     def __init__(self, client: ToolkitClient, dry_run: bool, skip_on_missing_ref: bool = False) -> None:
-        self.client = client
+        super().__init__(client)
         self.dry_run = dry_run
         self.skip_on_missing_ref = skip_on_missing_ref
 
-    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[tuple[IndustrialCanvasApply | None, MigrationIssue]]:
+    def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[IndustrialCanvasApply | None]:
         self._populate_cache(source)
-        output: list[tuple[IndustrialCanvasApply | None, MigrationIssue]] = []
+        output: list[IndustrialCanvasApply | None] = []
+        issues: list[CanvasMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.as_id()
+
+            if issue.missing_reference_ids:
+                self.logger.tracker.add_issue(identifier, "Missing reference IDs")
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     @property
@@ -340,7 +395,9 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
 
     def _map_single_item(self, canvas: IndustrialCanvas) -> tuple[IndustrialCanvasApply | None, CanvasMigrationIssue]:
         update = canvas.as_write()
-        issue = CanvasMigrationIssue(canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name)
+        issue = CanvasMigrationIssue(
+            canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name, id=canvas.canvas.name
+        )
 
         remaining_container_references: list[ContainerReferenceApply] = []
         new_fdm_references: list[FdmInstanceContainerReferenceApply] = []
@@ -399,17 +456,26 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
 
 
 class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(
-        self, source: Sequence[ThreeDModelResponse]
-    ) -> Sequence[tuple[ThreeDMigrationRequest | None, MigrationIssue]]:
+    def map(self, source: Sequence[ThreeDModelResponse]) -> Sequence[ThreeDMigrationRequest | None]:
         self._populate_cache(source)
-        output: list[tuple[ThreeDMigrationRequest | None, MigrationIssue]] = []
+        output: list[ThreeDMigrationRequest | None] = []
+        issues: list[ThreeDModelMigrationIssue] = []
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = item.name
+
+            if issue.error_message:
+                for error in issue.error_message:
+                    self.logger.tracker.add_issue(identifier, error)
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[ThreeDModelResponse]) -> None:
@@ -422,7 +488,7 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
     def _map_single_item(
         self, item: ThreeDModelResponse
     ) -> tuple[ThreeDMigrationRequest | None, ThreeDModelMigrationIssue]:
-        issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id)
+        issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id, id=item.name)
         instance_space: str | None = None
         last_revision_id: int | None = None
         model_type: Literal["CAD", "PointCloud", "Image360"] | None = None
@@ -478,17 +544,27 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
 
 
 class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingClassicResponse, AssetMappingDMRequest]):
-    def __init__(self, client: ToolkitClient) -> None:
-        self.client = client
-
-    def map(
-        self, source: Sequence[AssetMappingClassicResponse]
-    ) -> Sequence[tuple[AssetMappingDMRequest | None, MigrationIssue]]:
-        output: list[tuple[AssetMappingDMRequest | None, MigrationIssue]] = []
+    def map(self, source: Sequence[AssetMappingClassicResponse]) -> Sequence[AssetMappingDMRequest | None]:
+        output: list[AssetMappingDMRequest | None] = []
+        issues: list[ThreeDModelMigrationIssue] = []
         self._populate_cache(source)
         for item in source:
             mapped_item, issue = self._map_single_item(item)
-            output.append((mapped_item, issue))
+            identifier = f"AssetMapping_{item.model_id!s}_{item.revision_id!s}_{item.asset_id!s}"
+
+            if issue.error_message:
+                for error in issue.error_message:
+                    self.logger.tracker.add_issue(identifier, error)
+
+            if issue.has_issues:
+                issues.append(issue)
+
+            if mapped_item is None:
+                self.logger.tracker.finalize_item(identifier, "failure")
+
+            output.append(mapped_item)
+        if issues:
+            self.logger.log(issues)
         return output
 
     def _populate_cache(self, source: Sequence[AssetMappingClassicResponse]) -> None:
@@ -501,7 +577,9 @@ class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingClassicResponse,
     def _map_single_item(
         self, item: AssetMappingClassicResponse
     ) -> tuple[AssetMappingDMRequest | None, ThreeDModelMigrationIssue]:
-        issue = ThreeDModelMigrationIssue(model_name=f"AssetMapping_{item.model_id}", model_id=item.model_id)
+        issue = ThreeDModelMigrationIssue(
+            model_name=f"AssetMapping_{item.model_id}", model_id=item.model_id, id=f"AssetMapping_{item.model_id}"
+        )
         asset_instance_id = item.asset_instance_id
         if item.asset_id and asset_instance_id is None:
             asset_node_id = self.client.migration.lookup.assets(item.asset_id)

cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -1,34 +1,27 @@
-import json
-from typing import Any, ClassVar
+from typing import Any, Literal
 
-from cognite.client.data_classes.data_modeling import NodeId
-from cognite.client.utils._identifier import InstanceId
-from cognite.client.utils._text import to_camel_case
 from pydantic import BaseModel, Field, field_serializer
+from pydantic.alias_generators import to_camel
 
+from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import AssetCentricId
-from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+from cognite_toolkit._cdf_tk.storageio.logger import LogEntry
 
 
-class MigrationObject(BaseModel, alias_generator=to_camel_case, extra="ignore", populate_by_name=True): ...
-
-
-class MigrationIssue(MigrationObject):
+class MigrationIssue(LogEntry):
     """Represents an issue encountered during migration."""
 
-    type: ClassVar[str]
-
-    def dump(self) -> dict[str, JsonVal]:
-        # Dump json to ensure it is serializable
-        dumped = json.loads(self.model_dump_json(by_alias=True))
-        dumped["type"] = self.type
-        return dumped
+    type: str
 
     @property
     def has_issues(self) -> bool:
         """Check if there are any issues recorded in this MigrationIssue."""
         return True
 
+    def dump(self) -> dict[str, Any]:
+        """Serialize the MigrationIssue to a dictionary."""
+        return self.model_dump(by_alias=True)
+
 
 class ThreeDModelMigrationIssue(MigrationIssue):
     """Represents a 3D model migration issue encountered during migration.
@@ -37,7 +30,7 @@ class ThreeDModelMigrationIssue(MigrationIssue):
         model_external_id (str): The external ID of the 3D model that could not be migrated.
     """
 
-    type: ClassVar[str] = "threeDModelMigration"
+    type: Literal["threeDModelMigration"] = "threeDModelMigration"
     model_name: str
     model_id: int
     error_message: list[str] = Field(default_factory=list)
@@ -55,7 +48,7 @@ class ChartMigrationIssue(MigrationIssue):
         chart_external_id (str): The external ID of the chart that could not be migrated.
     """
 
-    type: ClassVar[str] = "chartMigration"
+    type: Literal["chartMigration"] = "chartMigration"
     chart_external_id: str
     missing_timeseries_ids: list[int] = Field(default_factory=list)
     missing_timeseries_external_ids: list[str] = Field(default_factory=list)
@@ -70,7 +63,7 @@ class ChartMigrationIssue(MigrationIssue):
 
 
 class CanvasMigrationIssue(MigrationIssue):
-    type: ClassVar[str] = "canvasMigration"
+    type: Literal["canvasMigration"] = "canvasMigration"
     canvas_external_id: str
     canvas_name: str
     missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
@@ -84,7 +77,7 @@ class CanvasMigrationIssue(MigrationIssue):
 class ReadIssue(MigrationIssue):
     """Represents a read issue encountered during migration."""
 
-    type: ClassVar[str] = "read"
+    ...
 
 
 class ReadFileIssue(ReadIssue):
@@ -95,7 +88,7 @@ class ReadFileIssue(ReadIssue):
         error (str | None): An optional error message providing additional details about the read issue.
     """
 
-    type: ClassVar[str] = "fileRead"
+    type: Literal["fileRead"] = "fileRead"
 
     row_no: int
     error: str | None = None
@@ -109,7 +102,7 @@ class ReadAPIIssue(ReadIssue):
         error (str | None): An optional error message providing additional details about the read issue.
     """
 
-    type: ClassVar[str] = "apiRead"
+    type: Literal["apiRead"] = "apiRead"
     asset_centric_id: AssetCentricId
     error: str | None = None
 
@@ -121,7 +114,7 @@ class ReadAPIIssue(ReadIssue):
         }
 
 
-class FailedConversion(MigrationObject):
+class FailedConversion(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
     """Represents a property that failed to convert during migration.
 
     Attributes:
@@ -135,7 +128,7 @@ class FailedConversion(MigrationObject):
     error: str
 
 
-class InvalidPropertyDataType(MigrationObject):
+class InvalidPropertyDataType(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
     """Represents a property with an invalid type during migration.
 
     Attributes:
@@ -160,9 +153,9 @@ class ConversionIssue(MigrationIssue):
         failed_conversions (list[FailedConversion]): List of properties that failed to convert with reasons.
     """
 
-    type: ClassVar[str] = "conversion"
+    type: Literal["conversion"] = "conversion"
     asset_centric_id: AssetCentricId
-    instance_id: InstanceId
+    instance_id: NodeReference
     missing_asset_centric_properties: list[str] = Field(default_factory=list)
     missing_instance_properties: list[str] = Field(default_factory=list)
     invalid_instance_property_types: list[InvalidPropertyDataType] = Field(default_factory=list)
@@ -181,10 +174,6 @@ class ConversionIssue(MigrationIssue):
             or self.missing_instance_space
         )
 
-    @field_serializer("instance_id")
-    def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
-        return instance_id.dump(include_instance_type=True)
-
     @field_serializer("asset_centric_id")
     def serialize_asset_centric_id(self, asset_centric_id: AssetCentricId) -> dict[str, Any]:
         return {
@@ -197,16 +186,10 @@ class WriteIssue(MigrationIssue):
     """Represents a write issue encountered during migration.
 
     Attributes:
-        instance_id (InstanceId): The InstanceId of the data model instance that could not be written.
         status_code (int): The HTTP status code returned during the write operation.
         message (str | None): An optional message providing additional details about the write issue.
     """
 
-    type: ClassVar[str] = "write"
-    instance_id: InstanceId
+    type: Literal["write"] = "write"
     status_code: int
     message: str | None = None
-
-    @field_serializer("instance_id")
-    def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
-        return instance_id.dump(include_instance_type=True)

cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -78,7 +78,7 @@ class AssetCentricMigrationIO(
         self.skip_linking = skip_linking
 
     def as_id(self, item: AssetCentricMapping) -> str:
-        return f"{item.mapping.resource_type}_{item.mapping.id}"
+        return str(item.mapping.as_asset_centric_id())
 
     def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterator[Page]:
         if isinstance(selector, MigrationCSVFileSelector):
@@ -181,7 +181,7 @@ class AssetCentricMigrationIO(
 
         pending_instance_id_endpoint = self.PENDING_INSTANCE_ID_ENDPOINT_BY_KIND[selector.kind]
         results: list[HTTPMessage] = []
-        to_upload = self.link_asset_centric(data_chunk, http_client, results, pending_instance_id_endpoint)
+        to_upload = self.link_asset_centric(data_chunk, http_client, pending_instance_id_endpoint)
         if to_upload:
             results.extend(list(super().upload_items(to_upload, http_client, None)))
         return results
@@ -191,7 +191,6 @@ class AssetCentricMigrationIO(
         cls,
         data_chunk: Sequence[UploadItem[InstanceApply]],
         http_client: HTTPClient,
-        results: list[HTTPMessage],
         pending_instance_id_endpoint: str,
     ) -> Sequence[UploadItem[InstanceApply]]:
         """Links asset-centric resources to their (uncreated) instances using the pending-instance-ids endpoint."""
@@ -212,7 +211,6 @@ class AssetCentricMigrationIO(
             for res in batch_results:
                 if isinstance(res, SuccessResponseItems):
                     successful_linked.update(res.ids)
-            results.extend(batch_results)
         to_upload = [item for item in data_chunk if item.source_id in successful_linked]
         return to_upload
 
@@ -393,7 +391,7 @@ class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse,
         self.data_model_type = data_model_type
 
     def as_id(self, item: ThreeDModelResponse) -> str:
-        return f"{item.name}_{item.id!s}"
+        return item.name
 
     def _is_selected(self, item: ThreeDModelResponse, included_models: set[int] | None) -> bool:
         return self._is_correct_type(item) and (included_models is None or item.id in included_models)

cognite_toolkit/_cdf_tk/storageio/logger.py

@@ -14,7 +14,6 @@ class LogEntry(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_
     """Represents a log entry for tracking storage I/O operations."""
 
     id: str
-    message: str
 
 
 OperationStatus: TypeAlias = Literal["success", "failure", "unchanged", "pending"]

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.48
+      image: cognite/toolkit:0.7.49
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.48
+      image: cognite/toolkit:0.7.49
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.48"
+version = "0.7.49"
 
 
 [plugins]

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.7.48"
+__version__ = "0.7.49"

{cognite_toolkit-0.7.48.dist-info → cognite_toolkit-0.7.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.48
+Version: 0.7.49
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.48.dist-info → cognite_toolkit-0.7.49.dist-info}/RECORD

@@ -222,15 +222,15 @@ cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMou
 cognite_toolkit/_cdf_tk/commands/_download.py,sha256=dVddH9t7oGx1kdQ3CCYYQb96Uxxy-xC8Opph98lo46U,6869
 cognite_toolkit/_cdf_tk/commands/_import_cmd.py,sha256=RkJ7RZI6zxe0_1xrPB-iJhCVchurmIAChilx0_XMR6k,11141
 cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py,sha256=8ki04tJGH1dHdF2NtVF4HyhaC0XDDS7onrH_nvd9KtE,153
-cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=z5Gt8vJM6klzKpiVXsylTZzD79zQl4wgqF2Vg2s-gfU,14215
-cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=rTwfriVG7W8wssysymoXVbfFyifCxgZqH82oK0sbWuQ,17658
+cognite_toolkit/_cdf_tk/commands/_migrate/command.py,sha256=S5ZB7POjJ2ue3D54kW8TZLFuRJAilqwlLlELu3HGEMM,13579
+cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py,sha256=-sZktmsDIqkhNg-kg95Ty6eTm7FiyFRi1ZQLOkZ5x2w,17889
 cognite_toolkit/_cdf_tk/commands/_migrate/creators.py,sha256=Gp3CKruTxeSDS3HiWiLsQ4mN2SZ4BQB5xQ50o68GTs8,9616
 cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py,sha256=3RRi9qj4rTcuJCxIPXb4s47Mm03udRH3OC9b3rnYIAc,11686
-cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=mwpUHc-M5CS3xhXs0SyGY16UL7gxx7JwcuiFlpq9ipI,24537
+cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py,sha256=qRCXI68Fq4eU4UcUwbiQNt2FCgBT_nWKzzUiEtWzfzI,27707
 cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py,sha256=0lRyDRJ8zo00OngwWagQgHQIaemK4eufW9kbWwZ9Yo0,7901
 cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py,sha256=nH_wC0RzcI721sXwH_pThOxPv1nq4-oBd1rCDSGbr9Y,5555
-cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=sRle8p_3Nc9WqyTXjAIEyvDag0kKFzmvCqudWqJjdZU,7551
-cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=RrTFQErm8O1gtwrsBa0OcUAMez3xC82gcoJOnLm9sXY,26915
+cognite_toolkit/_cdf_tk/commands/_migrate/issues.py,sha256=SFkfLBCLI4nZBIWSRnyu6G4xcdS-csYJwzmzc4Jmzho,6988
+cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py,sha256=ClRvEzJDOft_oQJGlKpYt8wBRjrfJ_jtpVmRe7YO8a8,26801
 cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py,sha256=RfqaNoso5CyBwc-p6ckwcYqBfZXKhdJgdGIyd0TATaI,2635
 cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py,sha256=N1H_-rBpPUD6pbrlcofn1uEK1bA694EUXEe1zIXeqyo,2489
 cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=c42Na8r1Xcd1zSIJtU8eZMQTFBTzNmxtTIBLKBePzyA,43131
@@ -377,7 +377,7 @@ cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=sgO5TAD87gHn8gxTwRM3yn__
 cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=R1f2A9ZkQ-8O0b0kjhku1PNLJnTOlLMTb0tnGSYW-nU,19138
 cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=2mb9nTC63cn3r1B9YqA0wAmHcbJhMDiET78T7XY5Yco,10821
 cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=VqzAF79ON3hjrVRTBsSiSGK9B_Xmm-PWhFxvA0YKgPE,5162
-cognite_toolkit/_cdf_tk/storageio/logger.py,sha256=MeeIRplxE8SrP-rWPgoM12_VcchlfG54_5Cwa5-hyZE,5574
+cognite_toolkit/_cdf_tk/storageio/logger.py,sha256=RDQvauxSjtlVX-V5XGkFCtqiWpv4KdjJrEcPFVNqPF0,5557
 cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=5qqgeCrsmMLVsrRuYKxCKq1bNnvJkWeAsATpTKN1eC4,4394
 cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
 cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_8YNmvVROERxzG82qIZhU7SM,3065
@@ -430,14 +430,14 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
 cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
 cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
 cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
-cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=lfmpp-r0PD6SNxA7W4vyXbZJbYS1BdNhMl_LHK7uUA4,667
-cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=yQ00CPvH5uHg093kLp5mT9YwHKL0secE07jyg59asWU,2430
-cognite_toolkit/_resources/cdf.toml,sha256=lBU3mKRWKTW1NF2Q_8PU08lGP8tyFrBlk9P8OezhilU,475
-cognite_toolkit/_version.py,sha256=bTiPIxy2WmA-9Z8WjK-wrhrEltIBC0FGCskW2gVynPQ,23
+cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=fmuInpVjQiw7cSiCZf8auBTyCojatOWboPcCJsucvNM,667
+cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=-lTJS4LYINJ9Y2Tqr6D_A97CdBnhiAD6HEgWwKmJm6A,2430
+cognite_toolkit/_resources/cdf.toml,sha256=nwpyy2sEPEnElhxnXuWfknQVEf-3o__qVyh2ntpgZ5E,475
+cognite_toolkit/_version.py,sha256=ozK73q6JEJBpdtK_85pf9Qpr10X0T3Pso8xvHij8gJs,23
 cognite_toolkit/config.dev.yaml,sha256=M33FiIKdS3XKif-9vXniQ444GTZ-bLXV8aFH86u9iUQ,332
 cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
 cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
-cognite_toolkit-0.7.48.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
-cognite_toolkit-0.7.48.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
-cognite_toolkit-0.7.48.dist-info/METADATA,sha256=jVdGQBFmveYK_NgMkDICE25_GSBGGVhkpIwcdaA33hM,5026
-cognite_toolkit-0.7.48.dist-info/RECORD,,
+cognite_toolkit-0.7.49.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
+cognite_toolkit-0.7.49.dist-info/entry_points.txt,sha256=EtZ17K2mUjh-AY0QNU1CPIB_aDSSOdmtNI_4Fj967mA,84
+cognite_toolkit-0.7.49.dist-info/METADATA,sha256=hqUn0j9TfjCIp3DiBXADDE70AqkJ6MNokFeM0WcA5LU,5026
+cognite_toolkit-0.7.49.dist-info/RECORD,,