cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py

@@ -0,0 +1,229 @@
+from cognite.client import data_modeling as dm
+from cognite.client.data_classes.data_modeling.containers import BTreeIndex
+
+SPACE = dm.SpaceApply(
+    "cognite_migration", description="Space for the asset-centric to data modeling migration", name="cdf_migration"
+)
+
+RESOURCE_VIEW_MAPPING = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="ResourceViewMapping",
+    used_for="node",
+    properties={
+        "resourceType": dm.ContainerProperty(
+            type=dm.Text(max_text_size=255),
+            nullable=False,
+        ),
+        "viewId": dm.ContainerProperty(
+            type=dm.data_types.Json(),
+            nullable=False,
+        ),
+        "propertyMapping": dm.ContainerProperty(
+            type=dm.data_types.Json(),
+            nullable=False,
+        ),
+    },
+)
+
+INSTANCE_SOURCE_CONTAINER = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="InstanceSource",
+    used_for="node",
+    properties={
+        "resourceType": dm.ContainerProperty(
+            type=dm.data_types.Enum(
+                values={
+                    "timeseries": dm.data_types.EnumValue(),
+                    "asset": dm.data_types.EnumValue(),
+                    "file": dm.data_types.EnumValue(),
+                    "event": dm.data_types.EnumValue(),
+                    "sequence": dm.data_types.EnumValue(),
+                }
+            ),
+            nullable=False,
+        ),
+        "id": dm.ContainerProperty(
+            type=dm.data_types.Int64(),
+            nullable=False,
+        ),
+        "dataSetId": dm.ContainerProperty(
+            type=dm.data_types.Int64(),
+            nullable=True,
+        ),
+        "classicExternalId": dm.ContainerProperty(
+            type=dm.data_types.Text(),
+            nullable=True,
+        ),
+        "preferredConsumerViewId": dm.ContainerProperty(
+            type=dm.data_types.Json(),
+            nullable=True,
+        ),
+        "resourceViewMapping": dm.ContainerProperty(
+            type=dm.data_types.DirectRelation(container=RESOURCE_VIEW_MAPPING.as_id()), nullable=True
+        ),
+    },
+    indexes={
+        "id": BTreeIndex(["id"], cursorable=True),
+        "resourceType": BTreeIndex(["resourceType", "id"], cursorable=False),
+    },
+)
+
+CREATED_SOURCE_SYSTEM = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="CreatedSourceSystem",
+    used_for="node",
+    properties={
+        "source": dm.ContainerProperty(
+            type=dm.Text(max_text_size=128),
+            nullable=False,
+        )
+    },
+    constraints={"sourceUnique": dm.UniquenessConstraint(["source"])},
+    indexes={
+        "source": BTreeIndex(["source"], cursorable=True),
+    },
+)
+
+SPACE_SOURCE = dm.ContainerApply(
+    space=SPACE.space,
+    external_id="SpaceSource",
+    used_for="node",
+    properties={
+        "instanceSpace": dm.ContainerProperty(
+            type=dm.Text(max_text_size=64),
+            nullable=False,
+        ),
+        "dataSetId": dm.ContainerProperty(
+            type=dm.data_types.Int64(),
+            nullable=False,
+        ),
+        "dataSetExternalId": dm.ContainerProperty(
+            type=dm.Text(max_text_size=256),
+            nullable=True,
+        ),
+    },
+    indexes={
+        "space": BTreeIndex(["instanceSpace"], cursorable=True),
+        "dataSetId": BTreeIndex(["dataSetId"], cursorable=True),
+        "dataSetExternalId": BTreeIndex(["dataSetExternalId"], cursorable=True),
+    },
+    constraints={
+        "dataSetIdUnique": dm.UniquenessConstraint(["dataSetId"]),
+    },
+)
+
+CONTAINERS = [RESOURCE_VIEW_MAPPING, INSTANCE_SOURCE_CONTAINER, CREATED_SOURCE_SYSTEM, SPACE_SOURCE]
+
+RESOURCE_VIEW_MAPPING_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="ResourceViewMapping",
+    version="v1",
+    name="ResourceViewMapping",
+    description="The mapping from asset-centric resources to data modeling view.",
+    properties={
+        "resourceType": dm.MappedPropertyApply(
+            container=RESOURCE_VIEW_MAPPING.as_id(),
+            container_property_identifier="resourceType",
+        ),
+        "viewId": dm.MappedPropertyApply(
+            container=RESOURCE_VIEW_MAPPING.as_id(),
+            container_property_identifier="viewId",
+        ),
+        "propertyMapping": dm.MappedPropertyApply(
+            container=RESOURCE_VIEW_MAPPING.as_id(),
+            container_property_identifier="propertyMapping",
+        ),
+    },
+)
+
+INSTANCE_SOURCE_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="InstanceSource",
+    version="v1",
+    name="InstanceSource",
+    description="The source of the instance in asset-centric resources.",
+    properties={
+        "resourceType": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="resourceType",
+        ),
+        "id": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="id",
+        ),
+        "dataSetId": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="dataSetId",
+        ),
+        "classicExternalId": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="classicExternalId",
+        ),
+        "preferredConsumerViewId": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="preferredConsumerViewId",
+        ),
+        "resourceViewMapping": dm.MappedPropertyApply(
+            container=INSTANCE_SOURCE_CONTAINER.as_id(),
+            container_property_identifier="resourceViewMapping",
+            source=RESOURCE_VIEW_MAPPING_VIEW.as_id(),
+        ),
+    },
+)
+
+CREATED_SOURCE_SYSTEM_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="CreatedSourceSystem",
+    version="v1",
+    name="CreatedSourceSystem",
+    description="The source string the SourceSystem was created from.",
+    properties={
+        "source": dm.MappedPropertyApply(
+            container=CREATED_SOURCE_SYSTEM.as_id(),
+            container_property_identifier="source",
+        ),
+    },
+)
+
+SPACE_SOURCE_VIEW = dm.ViewApply(
+    space=SPACE.space,
+    external_id="SpaceSource",
+    version="v1",
+    name="SpaceSource",
+    description="The mapping from CDF spaces to data sets.",
+    properties={
+        "instanceSpace": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="instanceSpace",
+            description="The identifier of the created instances space.",
+        ),
+        "dataSetId": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="dataSetId",
+            description="The dataSetId the space was created from.",
+        ),
+        "dataSetExternalId": dm.MappedPropertyApply(
+            container=SPACE_SOURCE.as_id(),
+            container_property_identifier="dataSetExternalId",
+            description="The externalId of the dataSet (if present) the space was created from.",
+        ),
+    },
+)
+
+INSTANCE_SOURCE_VIEW_ID = INSTANCE_SOURCE_VIEW.as_id()
+RESOURCE_VIEW_MAPPING_VIEW_ID = RESOURCE_VIEW_MAPPING_VIEW.as_id()
+CREATED_SOURCE_SYSTEM_VIEW_ID = CREATED_SOURCE_SYSTEM_VIEW.as_id()
+SPACE_SOURCE_VIEW_ID = SPACE_SOURCE_VIEW.as_id()
+
+VIEWS = [RESOURCE_VIEW_MAPPING_VIEW, INSTANCE_SOURCE_VIEW, CREATED_SOURCE_SYSTEM_VIEW, SPACE_SOURCE_VIEW]
+
+COGNITE_MIGRATION_MODEL = dm.DataModelApply(
+    space=SPACE.space,
+    external_id="CogniteMigration",
+    version="v1",
+    name="CDF Migration Model",
+    description="Data model for migrating asset-centric resources to data modeling resources in CDF.",
+    views=[INSTANCE_SOURCE_VIEW_ID, RESOURCE_VIEW_MAPPING_VIEW_ID, CREATED_SOURCE_SYSTEM_VIEW_ID, SPACE_SOURCE_VIEW_ID],
+)
+
+MODEL_ID = COGNITE_MIGRATION_MODEL.as_id()

cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py

@@ -0,0 +1,137 @@
+from functools import lru_cache
+
+from cognite.client.data_classes.data_modeling import ViewId
+
+from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
+
+ASSET_ID = "cdf_asset_mapping"
+EVENT_ID = "cdf_event_mapping"
+TIME_SERIES_ID = "cdf_time_series_mapping"
+TIME_SERIES_EXTRACTOR_ID = "cdf_time_series_extractor_mapping"
+FILE_METADATA_ID = "cdf_file_metadata_mapping"
+FILE_METADATA_EXTRACTOR_ID = "cdf_file_metadata_extractor_mapping"
+FILE_ANNOTATIONS_ID = "cdf_file_annotations_mapping"
+ASSET_ANNOTATIONS_ID = "cdf_asset_annotations_mapping"
+
+
+@lru_cache(maxsize=1)
+def create_default_mappings() -> list[ResourceViewMappingApply]:
+    """Return the default mappings for migration."""
+    ts_property_mapping = {
+        "name": "name",
+        "description": "description",
+        "isStep": "isStep",
+        "isString": "type",
+        "unit": "sourceUnit",
+        "unitExternalId": "unit",
+        "assetId": "assets",
+    }
+    file_property_mapping = {
+        "name": "name",
+        "source": "source",
+        "labels": "tags",
+        "mimeType": "mimeType",
+        "directory": "directory",
+        "assetIds": "assets",
+        "sourceCreatedTime": "sourceCreatedTime",
+        "sourceModifiedTime": "sourceUpdatedTime",
+    }
+
+    return [
+        ResourceViewMappingApply(
+            external_id=ASSET_ID,
+            resource_type="asset",
+            view_id=ViewId("cdf_cdm", "CogniteAsset", "v1"),
+            property_mapping={
+                "name": "name",
+                "description": "description",
+                "source": "source",
+                "labels": "tags",
+                "parentId": "parent",
+            },
+        ),
+        ResourceViewMappingApply(
+            external_id=EVENT_ID,
+            resource_type="event",
+            view_id=ViewId("cdf_cdm", "CogniteActivity", "v1"),
+            property_mapping={
+                "startTime": "startTime",
+                "endTime": "endTime",
+                "description": "description",
+                "source": "source",
+                "labels": "tags",
+                "assetIds": "assets",
+            },
+        ),
+        ResourceViewMappingApply(
+            external_id=TIME_SERIES_ID,
+            resource_type="timeseries",
+            view_id=ViewId("cdf_cdm", "CogniteTimeSeries", "v1"),
+            property_mapping=ts_property_mapping,
+        ),
+        ResourceViewMappingApply(
+            external_id=TIME_SERIES_EXTRACTOR_ID,
+            resource_type="timeseries",
+            view_id=ViewId("cdf_extraction_extensions", "CogniteExtractorTimeSeries", "v1"),
+            property_mapping={**ts_property_mapping, "metadata": "extractedData"},
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_METADATA_ID,
+            resource_type="file",
+            view_id=ViewId("cdf_cdm", "CogniteFile", "v1"),
+            property_mapping=file_property_mapping,
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_METADATA_EXTRACTOR_ID,
+            resource_type="file",
+            view_id=ViewId("cdf_extraction_extensions", "CogniteExtractorFile", "v1"),
+            property_mapping={**file_property_mapping, "metadata": "extractedData"},
+        ),
+        ResourceViewMappingApply(
+            external_id=ASSET_ANNOTATIONS_ID,
+            resource_type="assetAnnotation",
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
+            property_mapping={
+                # We are ignoring the symbol region in the default mapping.
+                "annotatedResourceId": "edge.startNode",
+                "annotationType": "edge.type.externalId",
+                "creatingUser": "sourceCreatedUser",
+                "creatingApp": "sourceId",
+                "creatingAppVersion": "sourceContext",
+                "status": "status",
+                "data.assetRef.id": "edge.endNode",
+                "data.assetRef.externalId": "edge.endNode",
+                "data.description": "description",
+                "data.pageNumber": "startNodePageNumber",
+                "data.textRegion.confidence": "confidence",
+                "data.textRegion.xMin": "startNodeXMin",
+                "data.textRegion.xMax": "startNodeXMax",
+                "data.textRegion.yMin": "startNodeYMin",
+                "data.textRegion.yMax": "startNodeYMax",
+                "data.text": "startNodeText",
+            },
+        ),
+        ResourceViewMappingApply(
+            external_id=FILE_ANNOTATIONS_ID,
+            resource_type="fileAnnotation",
+            view_id=ViewId("cdf_cdm", "CogniteDiagramAnnotation", "v1"),
+            property_mapping={
+                "annotatedResourceId": "edge.startNode",
+                "annotationType": "edge.type.externalId",
+                "creatingUser": "sourceCreatedUser",
+                "creatingApp": "sourceId",
+                "creatingAppVersion": "sourceContext",
+                "status": "status",
+                "data.fileRef.id": "edge.endNode",
+                "data.fileRef.externalId": "edge.endNode",
+                "data.description": "description",
+                "data.pageNumber": "startNodePageNumber",
+                "data.textRegion.confidence": "confidence",
+                "data.textRegion.xMin": "startNodeXMin",
+                "data.textRegion.xMax": "startNodeXMax",
+                "data.textRegion.yMin": "startNodeYMin",
+                "data.textRegion.yMax": "startNodeYMax",
+                "data.text": "startNodeText",
+            },
+        ),
+    ]

cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -0,0 +1,212 @@
+import json
+from typing import Any, ClassVar
+
+from cognite.client.data_classes.data_modeling import NodeId
+from cognite.client.utils._identifier import InstanceId
+from cognite.client.utils._text import to_camel_case
+from pydantic import BaseModel, Field, field_serializer
+
+from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
+from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+
+
+class MigrationObject(BaseModel, alias_generator=to_camel_case, extra="ignore", populate_by_name=True): ...
+
+
+class MigrationIssue(MigrationObject):
+    """Represents an issue encountered during migration."""
+
+    type: ClassVar[str]
+
+    def dump(self) -> dict[str, JsonVal]:
+        # Dump json to ensure it is serializable
+        dumped = json.loads(self.model_dump_json(by_alias=True))
+        dumped["type"] = self.type
+        return dumped
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this MigrationIssue."""
+        return True
+
+
+class ThreeDModelMigrationIssue(MigrationIssue):
+    """Represents a 3D model migration issue encountered during migration.
+
+    Attributes:
+        model_external_id (str): The external ID of the 3D model that could not be migrated.
+    """
+
+    type: ClassVar[str] = "threeDModelMigration"
+    model_name: str
+    model_id: int
+    error_message: list[str] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ThreeDModelMigrationIssue."""
+        return bool(self.error_message)
+
+
+class ChartMigrationIssue(MigrationIssue):
+    """Represents a chart migration issue encountered during migration.
+
+    Attributes:
+        chart_external_id (str): The external ID of the chart that could not be migrated.
+    """
+
+    type: ClassVar[str] = "chartMigration"
+    chart_external_id: str
+    missing_timeseries_ids: list[int] = Field(default_factory=list)
+    missing_timeseries_external_ids: list[str] = Field(default_factory=list)
+    missing_timeseries_identifier: list[str] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ChartMigrationIssue."""
+        return bool(
+            self.missing_timeseries_ids or self.missing_timeseries_external_ids or self.missing_timeseries_identifier
+        )
+
+
+class CanvasMigrationIssue(MigrationIssue):
+    type: ClassVar[str] = "canvasMigration"
+    canvas_external_id: str
+    canvas_name: str
+    missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this CanvasMigrationIssue."""
+        return bool(self.missing_reference_ids)
+
+
+class ReadIssue(MigrationIssue):
+    """Represents a read issue encountered during migration."""
+
+    type: ClassVar[str] = "read"
+
+
+class ReadFileIssue(ReadIssue):
+    """Represents a read issue encountered during migration of a file.
+
+    Attributes:
+        row_no (int): The row number in the CSV file where the issue occurred.
+        error (str | None): An optional error message providing additional details about the read issue.
+    """
+
+    type: ClassVar[str] = "fileRead"
+
+    row_no: int
+    error: str | None = None
+
+
+class ReadAPIIssue(ReadIssue):
+    """Represents a read issue encountered during migration from the API.
+
+    Attributes:
+        asset_centric_id (AssetCentricId): The identifier of the asset-centric resource that could not be read.
+        error (str | None): An optional error message providing additional details about the read issue.
+    """
+
+    type: ClassVar[str] = "apiRead"
+    asset_centric_id: AssetCentricId
+    error: str | None = None
+
+    @field_serializer("asset_centric_id")
+    def serialize_asset_centric_id(self, asset_centric_id: AssetCentricId) -> dict[str, Any]:
+        return {
+            "resourceType": asset_centric_id.resource_type,
+            "id": asset_centric_id.id_,
+        }
+
+
+class FailedConversion(MigrationObject):
+    """Represents a property that failed to convert during migration.
+
+    Attributes:
+        value (str | int | float | bool | None): The value that failed to convert
+        error (str): The error message explaining why the conversion failed.
+
+    """
+
+    property_id: str
+    value: str | int | float | bool | None | list | dict
+    error: str
+
+
+class InvalidPropertyDataType(MigrationObject):
+    """Represents a property with an invalid type during migration.
+
+    Attributes:
+        property_id (str): The identifier of the property in asset-centric.
+
+        expected_type (str): The expected type of the property.
+    """
+
+    property_id: str
+    expected_type: str
+
+
+class ConversionIssue(MigrationIssue):
+    """Represents a conversion issue encountered during migration.
+
+    Attributes:
+        asset_centric_id (AssetCentricId): The identifier of the asset-centric resource.
+        instance_id (InstanceId): The NodeId of the data model instance.
+        missing_asset_centric_properties (list[str]): List of source properties that are missing.
+        missing_instance_properties (list[str]): List of target properties that are missing.
+        invalid_instance_property_types (list[InvalidPropertyDataType]): List of properties with invalid types.
+        failed_conversions (list[FailedConversion]): List of properties that failed to convert with reasons.
+    """
+
+    type: ClassVar[str] = "conversion"
+    asset_centric_id: AssetCentricId
+    instance_id: InstanceId
+    missing_asset_centric_properties: list[str] = Field(default_factory=list)
+    missing_instance_properties: list[str] = Field(default_factory=list)
+    invalid_instance_property_types: list[InvalidPropertyDataType] = Field(default_factory=list)
+    failed_conversions: list[FailedConversion] = Field(default_factory=list)
+    ignored_asset_centric_properties: list[str] = Field(default_factory=list)
+    missing_instance_space: str | None = None
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ConversionIssue."""
+        return bool(
+            self.missing_asset_centric_properties
+            or self.missing_instance_properties
+            or self.invalid_instance_property_types
+            or self.failed_conversions
+            or self.missing_instance_space
+        )
+
+    @field_serializer("instance_id")
+    def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
+        return instance_id.dump(include_instance_type=True)
+
+    @field_serializer("asset_centric_id")
+    def serialize_asset_centric_id(self, asset_centric_id: AssetCentricId) -> dict[str, Any]:
+        return {
+            "resourceType": asset_centric_id.resource_type,
+            "id": asset_centric_id.id_,
+        }
+
+
+class WriteIssue(MigrationIssue):
+    """Represents a write issue encountered during migration.
+
+    Attributes:
+        instance_id (InstanceId): The InstanceId of the data model instance that could not be written.
+        status_code (int): The HTTP status code returned during the write operation.
+        message (str | None): An optional message providing additional details about the write issue.
+    """
+
+    type: ClassVar[str] = "write"
+    instance_id: InstanceId
+    status_code: int
+    message: str | None = None
+
+    @field_serializer("instance_id")
+    def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
+        return instance_id.dump(include_instance_type=True)