cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/command.py

@@ -0,0 +1,309 @@
+from collections.abc import Callable, Iterable, Sequence
+from enum import Enum
+from pathlib import Path
+
+from rich import print
+from rich.console import Console
+from rich.table import Table
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.commands._migrate.creators import MigrationCreator
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import DataMapper
+from cognite_toolkit._cdf_tk.commands.deploy import DeployCommand
+from cognite_toolkit._cdf_tk.constants import DMS_INSTANCE_LIMIT_MARGIN
+from cognite_toolkit._cdf_tk.cruds import ResourceWorker
+from cognite_toolkit._cdf_tk.data_classes import DeployResults
+from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitFileExistsError,
+    ToolkitMigrationError,
+    ToolkitValueError,
+)
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
+from cognite_toolkit._cdf_tk.storageio import T_Selector, UploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.utils import humanize_collection, safe_write, sanitize_filename
+from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
+from cognite_toolkit._cdf_tk.utils.fileio import Chunk, CSVWriter, NDJsonWriter, SchemaColumn, Uncompressed
+from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemMessage, SuccessResponseItems
+from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
+from cognite_toolkit._cdf_tk.utils.progress_tracker import AVAILABLE_STATUS, ProgressTracker, Status
+
+from .data_model import INSTANCE_SOURCE_VIEW_ID, MODEL_ID, RESOURCE_VIEW_MAPPING_VIEW_ID
+
+
+class MigrationCommand(ToolkitCommand):
+    class Steps(str, Enum):
+        DOWNLOAD = "download"
+        CONVERT = "convert"
+        UPLOAD = "upload"
+
+        @classmethod
+        def list(cls) -> list[str]:
+            return [step.value for step in cls.__members__.values()]
+
+    def migrate(
+        self,
+        selected: T_Selector,
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        log_dir: Path,
+        dry_run: bool = False,
+        verbose: bool = False,
+    ) -> ProgressTracker[str]:
+        if log_dir.exists() and any(log_dir.iterdir()):
+            raise ToolkitFileExistsError(
+                f"Log directory {log_dir} already exists. Please remove it or choose another directory."
+            )
+        self.validate_migration_model_available(data.client)
+        log_dir.mkdir(parents=True, exist_ok=True)
+        mapper.prepare(selected)
+
+        iteration_count: int | None = None
+        total_items = data.count(selected)
+        if total_items is not None:
+            iteration_count = (total_items // data.CHUNK_SIZE) + (1 if total_items % data.CHUNK_SIZE > 0 else 0)
+            self.validate_available_capacity(data.client, total_items)
+
+        console = Console()
+        tracker = ProgressTracker[str](self.Steps.list())
+        with (
+            NDJsonWriter(log_dir, kind=f"{selected.kind}MigrationIssues", compression=Uncompressed) as log_file,
+            HTTPClient(config=data.client.config) as write_client,
+        ):
+            executor = ProducerWorkerExecutor[Sequence[T_ResourceResponse], Sequence[UploadItem[T_ResourceRequest]]](
+                download_iterable=self._download_iterable(selected, data, tracker),
+                process=self._convert(mapper, data, tracker, log_file),
+                write=self._upload(selected, write_client, data, tracker, log_file, dry_run),
+                iteration_count=iteration_count,
+                max_queue_size=10,
+                download_description=f"Downloading {selected.display_name}",
+                process_description="Converting",
+                write_description="Uploading",
+                console=console,
+                verbose=verbose,
+            )
+
+            executor.run()
+            total = executor.total_items
+
+        self._print_table(tracker.aggregate(), console)
+        self._print_csv(tracker, log_dir, f"{selected.kind}Items", console)
+        executor.raise_on_error()
+        action = "Would migrate" if dry_run else "Migrating"
+        console.print(f"{action} {total:,} {selected.display_name} to instances.")
+        return tracker
+
+    def _print_table(self, results: dict[tuple[str, Status], int], console: Console) -> None:
+        for step in self.Steps:
+            # We treat pending as failed for summary purposes
+            results[(step.value, "failed")] = results.get((step.value, "failed"), 0) + results.get(
+                (step.value, "pending"), 0
+            )
+
+        table = Table(title="Migration Summary", show_lines=True)
+        table.add_column("Status", style="cyan", no_wrap=True)
+        for step in self.Steps:
+            table.add_column(step.value.capitalize(), style="magenta")
+        for status in AVAILABLE_STATUS:
+            if status == "pending":
+                # Skip pending as we treat it as failed
+                continue
+            row = [status]
+            for step in self.Steps:
+                row.append(str(results.get((step.value, status), 0)))
+            table.add_row(*row)
+
+        console.print(table)
+
+    def _print_csv(self, tracker: ProgressTracker[str], log_dir: Path, kind: str, console: Console) -> None:
+        with CSVWriter(log_dir, kind=kind, compression=Uncompressed, columns=self._csv_columns()) as csv_file:
+            batch: list[Chunk] = []
+            steps = self.Steps.list()
+            for item_id, progress in tracker.result().items():
+                batch.append({"ID": str(item_id), **{step: progress[step] for step in steps}})
+                if len(batch) >= 1000:
+                    csv_file.write_chunks(batch)
+                    batch = []
+            if batch:
+                csv_file.write_chunks(batch)
+        console.print(f"Migration items written to {log_dir}")
+
+    @classmethod
+    def _csv_columns(cls) -> list[SchemaColumn]:
+        return [
+            SchemaColumn(name="ID", type="string"),
+            *(SchemaColumn(name=step, type="string") for step in cls.Steps.list()),
+        ]
+
+    def _download_iterable(
+        self,
+        selected: T_Selector,
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        tracker: ProgressTracker[str],
+    ) -> Iterable[Sequence[T_ResourceResponse]]:
+        for page in data.stream_data(selected):
+            for item in page.items:
+                tracker.set_progress(data.as_id(item), self.Steps.DOWNLOAD, "success")
+            yield page.items
+
+    def _convert(
+        self,
+        mapper: DataMapper[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        data: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        tracker: ProgressTracker[str],
+        log_file: NDJsonWriter,
+    ) -> Callable[[Sequence[T_ResourceResponse]], Sequence[UploadItem[T_ResourceRequest]]]:
+        def track_mapping(source: Sequence[T_ResourceResponse]) -> list[UploadItem[T_ResourceRequest]]:
+            mapped = mapper.map(source)
+            issues: list[Chunk] = []
+            targets: list[UploadItem[T_ResourceRequest]] = []
+
+            for (target, issue), item in zip(mapped, source):
+                id_ = data.as_id(item)
+                result: Status = "failed" if target is None else "success"
+                tracker.set_progress(id_, step=self.Steps.CONVERT, status=result)
+
+                if issue.has_issues:
+                    # MyPy fails to understand that dict[str, JsonVal] is a Chunk
+                    issues.append(issue.dump())  # type: ignore[arg-type]
+                if target is not None:
+                    targets.append(UploadItem(source_id=id_, item=target))
+            if issues:
+                log_file.write_chunks(issues)
+            return targets
+
+        return track_mapping
+
+    def _upload(
+        self,
+        selected: T_Selector,
+        write_client: HTTPClient,
+        target: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
+        tracker: ProgressTracker[str],
+        log_file: NDJsonWriter,
+        dry_run: bool,
+    ) -> Callable[[Sequence[UploadItem[T_ResourceRequest]]], None]:
+        def upload_items(data_item: Sequence[UploadItem[T_ResourceRequest]]) -> None:
+            if not data_item:
+                return None
+            responses: Sequence[HTTPMessage]
+            if dry_run:
+                responses = [
+                    SuccessResponseItems(
+                        status_code=200, body="", content=b"", ids=[item.source_id for item in data_item]
+                    )
+                ]
+            else:
+                responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
+
+            issues: list[Chunk] = []
+            for item in responses:
+                if isinstance(item, SuccessResponseItems):
+                    for success_id in item.ids:
+                        tracker.set_progress(success_id, step=self.Steps.UPLOAD, status="success")
+                elif isinstance(item, ItemMessage):
+                    for failed_id in item.ids:
+                        tracker.set_progress(failed_id, step=self.Steps.UPLOAD, status="failed")
+
+                if not isinstance(item, SuccessResponseItems):
+                    issues.append(item.dump())  # type: ignore[arg-type]
+            if issues:
+                log_file.write_chunks(issues)
+            return None
+
+        return upload_items
+
+    @staticmethod
+    def validate_migration_model_available(client: ToolkitClient) -> None:
+        models = client.data_modeling.data_models.retrieve([MODEL_ID], inline_views=False)
+        if not models:
+            raise ToolkitMigrationError(
+                f"The migration data model {MODEL_ID!r} does not exist. "
+                "Please run the `cdf migrate prepare` command to deploy the migration data model."
+            )
+        elif len(models) > 1:
+            raise ToolkitMigrationError(
+                f"Multiple migration models {MODEL_ID!r}. "
+                "Please delete the duplicate models before proceeding with the migration."
+            )
+        model = models[0]
+        missing_views = {INSTANCE_SOURCE_VIEW_ID, RESOURCE_VIEW_MAPPING_VIEW_ID} - set(model.views or [])
+        if missing_views:
+            raise ToolkitMigrationError(
+                f"Invalid migration model. Missing views {humanize_collection(missing_views)}. "
+                f"Please run the `cdf migrate prepare` command to deploy the migration data model."
+            )
+
+    def validate_available_capacity(self, client: ToolkitClient, instance_count: int) -> None:
+        """Validate that the project has enough capacity to accommodate the migration."""
+
+        stats = client.data_modeling.statistics.project()
+
+        available_capacity = stats.instances.instances_limit - stats.instances.instances
+        available_capacity_after = available_capacity - instance_count
+
+        if available_capacity_after < DMS_INSTANCE_LIMIT_MARGIN:
+            raise ToolkitValueError(
+                "Cannot proceed with migration, not enough instance capacity available. Total capacity after migration"
+                f" would be {available_capacity_after:,} instances, which is less than the required margin of"
+                f" {DMS_INSTANCE_LIMIT_MARGIN:,} instances. Please increase the instance capacity in your CDF project"
+                f" or delete some existing instances before proceeding with the migration of {instance_count:,} assets."
+            )
+        total_instances = stats.instances.instances + instance_count
+        self.console(
+            f"Project has enough capacity for migration. Total instances after migration: {total_instances:,}."
+        )
+
+    def create(
+        self,
+        client: ToolkitClient,
+        creator: MigrationCreator,
+        dry_run: bool,
+        output_dir: Path,
+        verbose: bool = False,
+    ) -> DeployResults:
+        """This method is used to create migration resource in CDF."""
+        self.validate_migration_model_available(client)
+
+        deploy_cmd = DeployCommand(self.print_warning, silent=self.silent)
+        deploy_cmd.tracker = self.tracker
+
+        crud_cls = creator.CRUD
+        resource_list = creator.create_resources()
+
+        results = DeployResults([], "deploy", dry_run=dry_run)
+        crud = crud_cls.create_loader(client)
+        worker = ResourceWorker(crud, "deploy")
+        local_by_id = {crud.get_id(item): (item.dump(), item) for item in resource_list}
+        worker.validate_access(local_by_id, is_dry_run=dry_run)
+        cdf_resources = crud.retrieve(list(local_by_id.keys()))
+        resources = worker.categorize_resources(local_by_id, cdf_resources, False, verbose)
+
+        if dry_run:
+            result = deploy_cmd.dry_run_deploy(resources, crud, False, False)
+        else:
+            result = deploy_cmd.actual_deploy(resources, crud)
+            if result.calculated_total > 0 and creator.HAS_LINEAGE:
+                store_count = creator.store_lineage(resource_list)
+                self.console(f"Stored lineage for {store_count:,} {creator.DISPLAY_NAME}.")
+
+        verb = "Would deploy" if dry_run else "Deploying"
+        self.console(f"{verb} {creator.DISPLAY_NAME} to CDF.")
+
+        resource_configs = creator.resource_configs(resource_list)
+        for config in resource_configs:
+            filepath = output_dir / crud_cls.folder_name / f"{sanitize_filename(config.filestem)}.{crud_cls.kind}.yaml"
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            safe_write(filepath, yaml_safe_dump(config.data))
+        self.console(
+            f"{len(resource_configs)} {crud_cls.kind} resource configurations written to {(output_dir / crud_cls.folder_name).as_posix()!r}"
+        )
+        self.console("It is recommended to add these files to a Toolkit governed module.")
+
+        if result:
+            results[result.name] = result
+
+        if results.has_counts:
+            print(results.counts_table())
+
+        return results

cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -0,0 +1,373 @@
+from collections.abc import Iterable, Mapping, Set
+from typing import Any, ClassVar, cast
+
+from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
+from cognite.client.data_classes.data_modeling import (
+    DirectRelation,
+    DirectRelationReference,
+    EdgeId,
+    MappedProperty,
+    NodeApply,
+    NodeId,
+)
+from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeOrEdgeData, PropertyValueWrite
+from cognite.client.data_classes.data_modeling.views import ViewProperty
+from cognite.client.utils._identifier import InstanceId
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.client.data_classes.migration import (
+    AssetCentricId,
+    ResourceViewMappingApply,
+)
+from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
+from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
+    asset_centric_convert_to_primary_property,
+    convert_to_primary_property,
+)
+from cognite_toolkit._cdf_tk.utils.useful_types import (
+    AssetCentricResourceExtended,
+    AssetCentricTypeExtended,
+)
+
+from .data_model import INSTANCE_SOURCE_VIEW_ID
+from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
+
+
+class DirectRelationCache:
+    """Cache for direct relation references to look up target of direct relations.
+
+    This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
+    """
+
+    class TableName:
+        ASSET_ID = "assetId"
+        SOURCE_NAME = "source"
+        FILE_ID = "fileId"
+        ASSET_EXTERNAL_ID = "assetExternalId"
+        FILE_EXTERNAL_ID = "fileExternalId"
+
+    ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
+        ("timeseries", "assetId"),
+        ("file", "assetIds"),
+        ("event", "assetIds"),
+        ("sequence", "assetId"),
+        ("annotation", "data.assetRef.id"),
+        ("asset", "parentId"),
+    }
+    SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
+        ("asset", "source"),
+        ("event", "source"),
+        ("file", "source"),
+    }
+    FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
+        ("annotation", "data.fileRef.id"),
+        ("annotation", "annotatedResourceId"),
+    }
+    ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
+    FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
+
+    def __init__(self, client: ToolkitClient) -> None:
+        self._client = client
+        self._cache_map: dict[
+            tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
+        ] = {}
+        # Constructing the cache map to be accessed by both table name and property id
+        for table_name, properties in [
+            (self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
+            (self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
+            (self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
+            (self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
+            (self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
+        ]:
+            cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
+            self._cache_map[table_name] = cache
+            for key in properties:
+                self._cache_map[key] = cache
+
+    def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
+        """Update the cache with direct relation references for the given asset-centric resources.
+
+        This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
+        """
+        asset_ids: set[int] = set()
+        source_ids: set[str] = set()
+        file_ids: set[int] = set()
+        asset_external_ids: set[str] = set()
+        file_external_ids: set[str] = set()
+        for resource in resources:
+            if isinstance(resource, Annotation):
+                if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
+                    file_ids.add(resource.annotated_resource_id)
+                if "assetRef" in resource.data:
+                    asset_ref = resource.data["assetRef"]
+                    if isinstance(asset_id := asset_ref.get("id"), int):
+                        asset_ids.add(asset_id)
+                    if isinstance(asset_external_id := asset_ref.get("externalId"), str):
+                        asset_external_ids.add(asset_external_id)
+                if "fileRef" in resource.data:
+                    file_ref = resource.data["fileRef"]
+                    if isinstance(file_id := file_ref.get("id"), int):
+                        file_ids.add(file_id)
+                    if isinstance(file_external_id := file_ref.get("externalId"), str):
+                        file_external_ids.add(file_external_id)
+            elif isinstance(resource, Asset):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.parent_id is not None:
+                    asset_ids.add(resource.parent_id)
+            elif isinstance(resource, FileMetadata):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, Event):
+                if resource.source:
+                    source_ids.add(resource.source)
+                if resource.asset_ids:
+                    asset_ids.update(resource.asset_ids)
+            elif isinstance(resource, TimeSeries):
+                if resource.asset_id is not None:
+                    asset_ids.add(resource.asset_id)
+        if asset_ids:
+            self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
+        if source_ids:
+            # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
+            cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
+            missing = set(source_ids) - set(cache.keys())
+            if missing:
+                source_systems = self._client.migration.created_source_system.retrieve(list(missing))
+                for source_system in source_systems:
+                    cache[source_system.source] = source_system.as_direct_relation_reference()
+        if file_ids:
+            self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
+        if asset_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
+                self.TableName.ASSET_EXTERNAL_ID,
+            )
+        if file_external_ids:
+            self._update_cache(
+                self._client.migration.lookup.files(external_id=list(file_external_ids)),
+                self.TableName.FILE_EXTERNAL_ID,
+            )
+
+    def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
+        cache = self._cache_map[table_name]
+        for identifier, instance_id in instance_id_by_id.items():
+            cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id)  # type: ignore[index]
+
+    def get_cache(
+        self, resource_type: AssetCentricTypeExtended, property_id: str
+    ) -> Mapping[str | int, DirectRelationReference] | None:
+        """Get the cache for the given resource type and property ID."""
+        return self._cache_map.get((resource_type, property_id))  # type: ignore[return-value]
+
+
+def asset_centric_to_dm(
+    resource: AssetCentricResourceExtended,
+    instance_id: InstanceId,
+    view_source: ResourceViewMappingApply,
+    view_properties: dict[str, ViewProperty],
+    direct_relation_cache: DirectRelationCache,
+) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
+    """Convert an asset-centric resource to a data model instance.
+
+    Args:
+        resource (CogniteResource): The asset-centric resource to convert.
+        instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
+        view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
+        view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
+        direct_relation_cache (DirectRelationCache): Cache for direct relation references.
+
+    Returns:
+        tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
+    """
+    resource_type = _lookup_resource_type(resource)
+    dumped = resource.dump()
+    try:
+        id_ = dumped.pop("id")
+    except KeyError as e:
+        raise ValueError("Resource must have an 'id' field.") from e
+    if not isinstance(id_, int):
+        raise TypeError(f"Resource 'id' field must be an int, got {type(id_)}.")
+    data_set_id = dumped.pop("dataSetId", None)
+    external_id = dumped.pop("externalId", None)
+
+    issue = ConversionIssue(asset_centric_id=AssetCentricId(resource_type, id_=id_), instance_id=instance_id)
+
+    properties = create_properties(
+        dumped,
+        view_properties,
+        view_source.property_mapping,
+        resource_type,
+        issue=issue,
+        direct_relation_cache=direct_relation_cache,
+    )
+    sources: list[NodeOrEdgeData] = []
+    if properties:
+        sources.append(NodeOrEdgeData(source=view_source.view_id, properties=properties))
+
+    if resource_type != "annotation":
+        instance_source_properties = {
+            "resourceType": resource_type,
+            "id": id_,
+            "dataSetId": data_set_id,
+            "classicExternalId": external_id,
+        }
+        sources.append(NodeOrEdgeData(source=INSTANCE_SOURCE_VIEW_ID, properties=instance_source_properties))
+
+    instance: NodeApply | EdgeApply
+    if isinstance(instance_id, EdgeId):
+        edge_properties = create_edge_properties(
+            dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
+        )
+        if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
+            # Failed conversion of edge properties
+            return None, issue
+        instance = EdgeApply(
+            space=instance_id.space,
+            external_id=instance_id.external_id,
+            sources=sources,
+            **edge_properties,  # type: ignore[arg-type]
+        )
+    elif isinstance(instance_id, NodeId):
+        instance = NodeApply(space=instance_id.space, external_id=instance_id.external_id, sources=sources)
+    else:
+        raise RuntimeError(f"Unexpected instance_id type {type(instance_id)}")
+
+    return instance, issue
+
+
+def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
+    if isinstance(resource_type, Asset):
+        return "asset"
+    elif isinstance(resource_type, FileMetadata):
+        return "file"
+    elif isinstance(resource_type, Event):
+        return "event"
+    elif isinstance(resource_type, TimeSeries):
+        return "timeseries"
+    elif isinstance(resource_type, Annotation):
+        if resource_type.annotated_resource_type == "file" and resource_type.annotation_type in (
+            "diagrams.AssetLink",
+            "diagrams.FileLink",
+        ):
+            return "annotation"
+    raise ValueError(f"Unsupported resource type: {resource_type}")
+
+
+def create_properties(
+    dumped: dict[str, Any],
+    view_properties: dict[str, ViewProperty],
+    property_mapping: dict[str, str],
+    resource_type: AssetCentricTypeExtended,
+    issue: ConversionIssue,
+    direct_relation_cache: DirectRelationCache,
+) -> dict[str, PropertyValueWrite]:
+    """
+    Create properties for a data model instance from an asset-centric resource.
+
+    Args:
+        dumped: Dict representation of the asset-centric resource.
+        view_properties: Defined properties referenced in the view source mapping.
+        property_mapping:  Mapping from asset-centric property IDs to data model property IDs.
+        resource_type: The type of the asset-centric resource (e.g., "asset", "timeseries").
+        issue: ConversionIssue object to log any issues encountered during conversion.
+        direct_relation_cache: Cache for direct relation references to look up target of direct relations.
+
+    Returns:
+        Dict of property IDs to PropertyValueWrite objects.
+
+    """
+    flatten_dump = flatten_dict_json_path(dumped, keep_structured=set(property_mapping.keys()))
+    properties: dict[str, PropertyValueWrite] = {}
+    ignored_asset_centric_properties: set[str] = set()
+    for prop_json_path, prop_id in property_mapping.items():
+        if prop_json_path not in flatten_dump:
+            continue
+        if prop_id not in view_properties:
+            continue
+        if prop_id in properties:
+            ignored_asset_centric_properties.add(prop_json_path)
+            continue
+        dm_prop = view_properties[prop_id]
+        if not isinstance(dm_prop, MappedProperty):
+            issue.invalid_instance_property_types.append(
+                InvalidPropertyDataType(property_id=prop_id, expected_type=MappedProperty.__name__)
+            )
+            continue
+        try:
+            value = asset_centric_convert_to_primary_property(
+                flatten_dump[prop_json_path],
+                dm_prop.type,
+                dm_prop.nullable,
+                destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
+                source_property=(resource_type, prop_json_path),
+                direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
+            )
+        except (ValueError, TypeError, NotImplementedError) as e:
+            issue.failed_conversions.append(
+                FailedConversion(property_id=prop_json_path, value=flatten_dump[prop_json_path], error=str(e))
+            )
+            continue
+        properties[prop_id] = value
+
+    issue.ignored_asset_centric_properties = sorted(
+        (set(flatten_dump.keys()) - set(property_mapping.keys())) | ignored_asset_centric_properties
+    )
+    issue.missing_asset_centric_properties = sorted(set(property_mapping.keys()) - set(flatten_dump.keys()))
+    # Node and edge properties are handled separately
+    issue.missing_instance_properties = sorted(
+        {
+            prop_id
+            for prop_id in property_mapping.values()
+            if not (prop_id.startswith("edge.") or prop_id.startswith("node."))
+        }
+        - set(view_properties.keys())
+    )
+    return properties
+
+
+def create_edge_properties(
+    dumped: dict[str, Any],
+    property_mapping: dict[str, str],
+    resource_type: AssetCentricTypeExtended,
+    issue: ConversionIssue,
+    direct_relation_cache: DirectRelationCache,
+    default_instance_space: str,
+) -> dict[str, DirectRelationReference]:
+    flatten_dump = flatten_dict_json_path(dumped)
+    edge_properties: dict[str, DirectRelationReference] = {}
+    for prop_json_path, prop_id in property_mapping.items():
+        if not prop_id.startswith("edge."):
+            continue
+        if prop_json_path not in flatten_dump:
+            continue
+        edge_prop_id = prop_id.removeprefix("edge.")
+        if edge_prop_id in ("startNode", "endNode", "type"):
+            # DirectRelation lookup.
+            try:
+                value = convert_to_primary_property(
+                    flatten_dump[prop_json_path],
+                    DirectRelation(),
+                    False,
+                    direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
+                )
+            except (ValueError, TypeError, NotImplementedError) as e:
+                issue.failed_conversions.append(
+                    FailedConversion(property_id=prop_json_path, value=flatten_dump[prop_json_path], error=str(e))
+                )
+                continue
+        elif edge_prop_id.endswith(".externalId"):
+            # Just an external ID string.
+            edge_prop_id = edge_prop_id.removesuffix(".externalId")
+            value = DirectRelationReference(default_instance_space, str(flatten_dump[prop_json_path]))
+        else:
+            issue.invalid_instance_property_types.append(
+                InvalidPropertyDataType(property_id=prop_id, expected_type="EdgeProperty")
+            )
+            continue
+        # We know that value is DirectRelationReference here
+        edge_properties[edge_prop_id.replace("Node", "_node")] = value  # type: ignore[assignment]
+
+    return edge_properties