cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py

@@ -67,7 +67,52 @@ INSTANCE_SOURCE_CONTAINER = dm.ContainerApply(
         "resourceType": BTreeIndex(["resourceType", "id"], cursorable=False),
     },
 )
-CONTAINERS = [RESOURCE_VIEW_MAPPING, INSTANCE_SOURCE_CONTAINER]
+
+CREATED_SOURCE_SYSTEM = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="CreatedSourceSystem",
+    used_for="node",
+    properties={
+        "source": dm.ContainerProperty(
+            type=dm.Text(max_text_size=128),
+            nullable=False,
+        )
+    },
+    constraints={"sourceUnique": dm.UniquenessConstraint(["source"])},
+    indexes={
+        "source": BTreeIndex(["source"], cursorable=True),
+    },
+)
+
+SPACE_SOURCE = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="SpaceSource",
+    used_for="node",
+    properties={
+        "instanceSpace": dm.ContainerProperty(
+            type=dm.Text(max_text_size=64),
+            nullable=False,
+        ),
+        "dataSetId": dm.ContainerProperty(
+            type=dm.data_types.Int64(),
+            nullable=False,
+        ),
+        "dataSetExternalId": dm.ContainerProperty(
+            type=dm.Text(max_text_size=256),
+            nullable=True,
+        ),
+    },
+    indexes={
+        "space": BTreeIndex(["instanceSpace"], cursorable=True),
+        "dataSetId": BTreeIndex(["dataSetId"], cursorable=True),
+        "dataSetExternalId": BTreeIndex(["dataSetExternalId"], cursorable=True),
+    },
+    constraints={
+        "dataSetIdUnique": dm.UniquenessConstraint(["dataSetId"]),
+    },
+)
+
+CONTAINERS = [RESOURCE_VIEW_MAPPING, INSTANCE_SOURCE_CONTAINER, CREATED_SOURCE_SYSTEM, SPACE_SOURCE]
 
 RESOURCE_VIEW_MAPPING_VIEW = dm.ViewApply(
     space=SPACE.space,
@@ -126,10 +171,51 @@ INSTANCE_SOURCE_VIEW = dm.ViewApply(
     },
 )
 
+CREATED_SOURCE_SYSTEM_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="CreatedSourceSystem",
+    version="v1",
+    name="CreatedSourceSystem",
+    description="The source string the SourceSystem was created from.",
+    properties={
+        "source": dm.MappedPropertyApply(
+            container=CREATED_SOURCE_SYSTEM.as_id(),
+            container_property_identifier="source",
+        ),
+    },
+)
+
+SPACE_SOURCE_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="SpaceSource",
+    version="v1",
+    name="SpaceSource",
+    description="The mapping from CDF spaces to data sets.",
+    properties={
+        "instanceSpace": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="instanceSpace",
+            description="The identifier of the created instances space.",
+        ),
+        "dataSetId": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="dataSetId",
+            description="The dataSetId the space was created from.",
+        ),
+        "dataSetExternalId": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="dataSetExternalId",
+            description="The externalId of the dataSet (if present) the space was created from.",
+        ),
+    },
+)
+
 INSTANCE_SOURCE_VIEW_ID = INSTANCE_SOURCE_VIEW.as_id()
 RESOURCE_VIEW_MAPPING_VIEW_ID = RESOURCE_VIEW_MAPPING_VIEW.as_id()
+CREATED_SOURCE_SYSTEM_VIEW_ID = CREATED_SOURCE_SYSTEM_VIEW.as_id()
+SPACE_SOURCE_VIEW_ID = SPACE_SOURCE_VIEW.as_id()
 
-VIEWS = [RESOURCE_VIEW_MAPPING_VIEW, INSTANCE_SOURCE_VIEW]
+VIEWS = [RESOURCE_VIEW_MAPPING_VIEW, INSTANCE_SOURCE_VIEW, CREATED_SOURCE_SYSTEM_VIEW, SPACE_SOURCE_VIEW]
 
 COGNITE_MIGRATION_MODEL = dm.DataModelApply(
     space=SPACE.space,
@@ -137,7 +223,7 @@ COGNITE_MIGRATION_MODEL = dm.DataModelApply(
     version="v1",
     name="CDF Migration Model",
     description="Data model for migrating asset-centric resources to data modeling resources in CDF.",
-    views=[RESOURCE_VIEW_MAPPING_VIEW.as_id(), INSTANCE_SOURCE_VIEW.as_id()],
+    views=[INSTANCE_SOURCE_VIEW_ID, RESOURCE_VIEW_MAPPING_VIEW_ID, CREATED_SOURCE_SYSTEM_VIEW_ID, SPACE_SOURCE_VIEW_ID],
 )
 
 MODEL_ID = COGNITE_MIGRATION_MODEL.as_id()

cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -4,18 +4,42 @@ from cognite.client.data_classes.data_modeling import ViewId
 
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
 
-_ASSET_ID = "cdf_asset_mapping"
-_EVENT_ID = "cdf_event_mapping"
-_TIME_SERIES_ID = "cdf_time_series_mapping"
-_FILE_METADATA_ID = "cdf_file_metadata_mapping"
+ASSET_ID = "cdf_asset_mapping"
+EVENT_ID = "cdf_event_mapping"
+TIME_SERIES_ID = "cdf_time_series_mapping"
+TIME_SERIES_EXTRACTOR_ID = "cdf_time_series_extractor_mapping"
+FILE_METADATA_ID = "cdf_file_metadata_mapping"
+FILE_METADATA_EXTRACTOR_ID = "cdf_file_metadata_extractor_mapping"
+FILE_ANNOTATIONS_ID = "cdf_file_annotations_mapping"
+ASSET_ANNOTATIONS_ID = "cdf_asset_annotations_mapping"
 
 
 @lru_cache(maxsize=1)
 def create_default_mappings() -> list[ResourceViewMappingApply]:
     """Return the default mappings for migration."""
+    ts_property_mapping = {
+        "name": "name",
+        "description": "description",
+        "isStep": "isStep",
+        "isString": "type",
+        "unit": "sourceUnit",
+        "unitExternalId": "unit",
+        "assetId": "assets",
+    }
+    file_property_mapping = {
+        "name": "name",
+        "source": "source",
+        "labels": "tags",
+        "mimeType": "mimeType",
+        "directory": "directory",
+        "assetIds": "assets",
+        "sourceCreatedTime": "sourceCreatedTime",
+        "sourceModifiedTime": "sourceUpdatedTime",
+    }
+
     return [
         ResourceViewMappingApply(
-            external_id=_ASSET_ID,
+            external_id=ASSET_ID,
             resource_type="asset",
             view_id=ViewId("cdf_cdm", "CogniteAsset", "v1"),
             property_mapping={
@@ -23,10 +47,11 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
                 "description": "description",
                 "source": "source",
                 "labels": "tags",
+                "parentId": "parent",
             },
         ),
         ResourceViewMappingApply(
-            external_id=_EVENT_ID,
+            external_id=EVENT_ID,
             resource_type="event",
             view_id=ViewId("cdf_cdm", "CogniteActivity", "v1"),
             property_mapping={
@@ -35,36 +60,78 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
                 "description": "description",
                 "source": "source",
                 "labels": "tags",
+                "assetIds": "assets",
             },
         ),
         ResourceViewMappingApply(
-            external_id=_TIME_SERIES_ID,
+            external_id=TIME_SERIES_ID,
             resource_type="timeseries",
             view_id=ViewId("cdf_cdm", "CogniteTimeSeries", "v1"),
-            property_mapping={
-                "name": "name",
-                "description": "description",
-                "isStep": "isStep",
-                "isString": "type",
-                "legacyName": "alias",
-                "unit": "sourceUnit",
-                "unitExternalId": "unit",
-                "source": "source",
-            },
+            property_mapping=ts_property_mapping,
         ),
         ResourceViewMappingApply(
-            external_id=_FILE_METADATA_ID,
+            external_id=TIME_SERIES_EXTRACTOR_ID,
+            resource_type="timeseries",
+            view_id=ViewId("cdf_extraction_extensions", "CogniteExtractorTimeSeries", "v1"),
+            property_mapping={**ts_property_mapping, "metadata": "extractedData"},
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_METADATA_ID,
             resource_type="file",
             view_id=ViewId("cdf_cdm", "CogniteFile", "v1"),
+            property_mapping=file_property_mapping,
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_METADATA_EXTRACTOR_ID,
+            resource_type="file",
+            view_id=ViewId("cdf_extraction_extensions", "CogniteExtractorFile", "v1"),
+            property_mapping={**file_property_mapping, "metadata": "extractedData"},
+        ),
+        ResourceViewMappingApply(
+            external_id=ASSET_ANNOTATIONS_ID,
+            resource_type="assetAnnotation",
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
             property_mapping={
-                "name": "name",
-                "description": "description",
-                "source": "source",
-                "labels": "tags",
-                "mimeType": "mimeType",
-                "directory": "directory",
-                "sourceCreatedTime": "sourceCreatedTime",
-                "sourceUpdatedTime": "sourceUpdatedTime",
+                # We are ignoring the symbol region in the default mapping.
+                "annotatedResourceId": "edge.startNode",
+                "annotationType": "edge.type.externalId",
+                "creatingUser": "sourceCreatedUser",
+                "creatingApp": "sourceId",
+                "creatingAppVersion": "sourceContext",
+                "status": "status",
+                "data.assetRef.id": "edge.endNode",
+                "data.assetRef.externalId": "edge.endNode",
+                "data.description": "description",
+                "data.pageNumber": "startNodePageNumber",
+                "data.textRegion.confidence": "confidence",
+                "data.textRegion.xMin": "startNodeXMin",
+                "data.textRegion.xMax": "startNodeXMax",
+                "data.textRegion.yMin": "startNodeYMin",
+                "data.textRegion.yMax": "startNodeYMax",
+                "data.text": "startNodeText",
+            },
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_ANNOTATIONS_ID,
+            resource_type="fileAnnotation",
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
+            property_mapping={
+                "annotatedResourceId": "edge.startNode",
+                "annotationType": "edge.type.externalId",
+                "creatingUser": "sourceCreatedUser",
+                "creatingApp": "sourceId",
+                "creatingAppVersion": "sourceContext",
+                "status": "status",
+                "data.fileRef.id": "edge.endNode",
+                "data.fileRef.externalId": "edge.endNode",
+                "data.description": "description",
+                "data.pageNumber": "startNodePageNumber",
+                "data.textRegion.confidence": "confidence",
+                "data.textRegion.xMin": "startNodeXMin",
+                "data.textRegion.xMax": "startNodeXMax",
+                "data.textRegion.yMin": "startNodeYMin",
+                "data.textRegion.yMax": "startNodeYMax",
+                "data.text": "startNodeText",
             },
         ),
     ]

cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -24,6 +24,44 @@ class MigrationIssue(MigrationObject):
         dumped["type"] = self.type
         return dumped
 
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this MigrationIssue."""
+        return True
+
+
+class ChartMigrationIssue(MigrationIssue):
+    """Represents a chart migration issue encountered during migration.
+
+    Attributes:
+        chart_external_id (str): The external ID of the chart that could not be migrated.
+    """
+
+    type: ClassVar[str] = "chartMigration"
+    chart_external_id: str
+    missing_timeseries_ids: list[int] = Field(default_factory=list)
+    missing_timeseries_external_ids: list[str] = Field(default_factory=list)
+    missing_timeseries_identifier: list[str] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ChartMigrationIssue."""
+        return bool(
+            self.missing_timeseries_ids or self.missing_timeseries_external_ids or self.missing_timeseries_identifier
+        )
+
+
+class CanvasMigrationIssue(MigrationIssue):
+    type: ClassVar[str] = "canvasMigration"
+    canvas_external_id: str
+    canvas_name: str
+    missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this CanvasMigrationIssue."""
+        return bool(self.missing_reference_ids)
+
 
 class ReadIssue(MigrationIssue):
     """Represents a read issue encountered during migration."""
@@ -112,6 +150,7 @@ class ConversionIssue(MigrationIssue):
     invalid_instance_property_types: list[InvalidPropertyDataType] = Field(default_factory=list)
     failed_conversions: list[FailedConversion] = Field(default_factory=list)
     ignored_asset_centric_properties: list[str] = Field(default_factory=list)
+    missing_instance_space: str | None = None
 
     @property
     def has_issues(self) -> bool:
@@ -121,6 +160,7 @@ class ConversionIssue(MigrationIssue):
             or self.missing_instance_properties
             or self.invalid_instance_property_types
             or self.failed_conversions
+            or self.missing_instance_space
         )
 
     @field_serializer("instance_id")