cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl

cognite/neat/_graph/loaders/_rdf2dms.py

@@ -4,15 +4,14 @@ import urllib.parse
 import warnings
 from collections import defaultdict
 from collections.abc import Iterable, Sequence
-from graphlib import TopologicalSorter
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, cast, get_args
+from typing import Any, Literal, cast, get_args, overload
 
 import yaml
 from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes.capabilities import Capability, DataModelInstancesAcl
-from cognite.client.data_classes.data_modeling import ViewId
 from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
 from cognite.client.data_classes.data_modeling.ids import InstanceId
 from cognite.client.data_classes.data_modeling.views import SingleEdgeConnection
@@ -21,251 +20,114 @@ from pydantic import BaseModel, ValidationInfo, create_model, field_validator
 from rdflib import RDF, URIRef
 
 from cognite.neat._client import NeatClient
+from cognite.neat._client._api_client import SchemaAPI
 from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT, is_readonly_property
-from cognite.neat._graph._tracking import LogTracker, Tracker
-from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
+from cognite.neat._issues import IssueList, NeatError, NeatIssue, catch_issues
 from cognite.neat._issues.errors import (
-    ResourceConversionError,
+    AuthorizationError,
     ResourceCreationError,
     ResourceDuplicatedError,
-    ResourceRetrievalError,
+    ResourceNotFoundError,
 )
-from cognite.neat._issues.warnings import PropertyDirectRelationLimitWarning, PropertyTypeNotSupportedWarning
-from cognite.neat._rules.analysis._dms import DMSAnalysis
+from cognite.neat._issues.warnings import (
+    NeatValueWarning,
+    PropertyDirectRelationLimitWarning,
+    PropertyMultipleValueWarning,
+    PropertyTypeNotSupportedWarning,
+    ResourceNeatWarning,
+)
+from cognite.neat._rules.analysis import RulesAnalysis
+from cognite.neat._rules.analysis._base import ViewQuery, ViewQueryDict
 from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json, String
-from cognite.neat._rules.models.entities._single_value import ViewEntity
+from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._shared import InstanceType
 from cognite.neat._store import NeatGraphStore
 from cognite.neat._utils.auxiliary import create_sha256_hash
 from cognite.neat._utils.collection_ import iterate_progress_bar_if_above_config_threshold
-from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._utils.rdf_ import namespace_as_space, remove_namespace_from_uri, split_uri
+from cognite.neat._utils.text import NamingStandardization, humanize_collection
 from cognite.neat._utils.upload import UploadResult
 
-from ._base import _END_OF_CLASS, CDFLoader
+from ._base import _END_OF_CLASS, _START_OF_CLASS, CDFLoader
+
+
+@dataclass
+class _ViewIterator:
+    """This is a helper class to iterate over the views
+
+    Args:
+        view_id: The view to iterate over
+        instance_count: The number of instances in the view
+        hierarchical_properties: The properties that are hierarchical, meaning they point to the same instances.
+        query: The query to get the instances from the store.
+        view: The view object from the client.
+    """
+
+    view_id: dm.ViewId
+    instance_count: int
+    hierarchical_properties: set[str]
+    query: ViewQuery
+    view: dm.View | None = None
+
+
+@dataclass
+class _Projection:
+    """This is a helper class to project triples to a node and/or edge(s)"""
+
+    view_id: dm.ViewId
+    used_for: Literal["node", "edge", "all"]
+    pydantic_cls: type[BaseModel]
+    edge_by_type: dict[str, tuple[str, dm.EdgeConnection]]
+    edge_by_prop_id: dict[str, tuple[str, dm.EdgeConnection]]
 
 
 class DMSLoader(CDFLoader[dm.InstanceApply]):
     """Loads Instances to Cognite Data Fusion Data Model Service from NeatGraph.
 
     Args:
-        graph_store (NeatGraphStore): The graph store to load the data into.
-        data_model (dm.DataModel[dm.View] | None): The data model to load.
+        dms_rules (DMSRules): The DMS rules used by the data model.
+        info_rules (InformationRules): The information rules used by the data model, used to
+            look+up the instances in the store.
+        graph_store (NeatGraphStore): The graph store to load the data from.
         instance_space (str): The instance space to load the data into.
-        class_neat_id_by_view_id (dict[ViewId, URIRef] | None): A mapping from view id to class name. Defaults to None.
         create_issues (Sequence[NeatIssue] | None): A list of issues that occurred during reading. Defaults to None.
-        tracker (type[Tracker] | None): The tracker to use. Defaults to None.
-        rules (DMSRules | None): The DMS rules used by the data model. This is used to lookup the
-            instances in the store. Defaults to None.
         client (NeatClient | None): This is used to lookup containers such that the loader
             creates instances in accordance with required constraints. Defaults to None.
+        unquote_external_ids (bool): If True, the loader will unquote external ids before creating the instances.
+        neat_prefix_by_predicate_uri (dict[URIRef, str] | None): A dictionary that maps a predicate URIRef to a
+            prefix that Neat added to the object upon extraction. This is used to remove the prefix from the
+            object before creating the instance.
     """
 
     def __init__(
         self,
+        dms_rules: DMSRules,
+        info_rules: InformationRules,
         graph_store: NeatGraphStore,
-        data_model: dm.DataModel[dm.View] | None,
         instance_space: str,
-        class_neat_id_by_view_id: dict[ViewId, URIRef] | None = None,
-        create_issues: Sequence[NeatIssue] | None = None,
-        tracker: type[Tracker] | None = None,
-        rules: DMSRules | None = None,
+        space_property: str | None = None,
+        use_source_space: bool = False,
         client: NeatClient | None = None,
+        create_issues: Sequence[NeatIssue] | None = None,
         unquote_external_ids: bool = False,
+        neat_prefix_by_predicate_uri: dict[URIRef, str] | None = None,
+        neat_prefix_by_type_uri: dict[URIRef, str] | None = None,
     ):
         super().__init__(graph_store)
-        self.data_model = data_model
-        self.instance_space = instance_space
-        self.class_neat_id_by_view_id = class_neat_id_by_view_id or {}
+        self.dms_rules = dms_rules
+        self.info_rules = info_rules
+        self.neat_prefix_by_predicate_uri = neat_prefix_by_predicate_uri or {}
+        self.neat_prefix_by_type_uri = neat_prefix_by_type_uri or {}
+        self._instance_space = instance_space
+        self._space_property = space_property
+        self._use_source_space = use_source_space
+        self._space_by_instance_uri: dict[URIRef, str] = defaultdict(lambda: instance_space)
+        self._external_id_by_uri: dict[URIRef, str] = {}
         self._issues = IssueList(create_issues or [])
-        self._tracker: type[Tracker] = tracker or LogTracker
-        self.rules = rules
         self._client = client
         self._unquote_external_ids = unquote_external_ids
 
-    @classmethod
-    def from_data_model_id(
-        cls,
-        client: NeatClient,
-        data_model_id: dm.DataModelId,
-        graph_store: NeatGraphStore,
-        instance_space: str,
-    ) -> "DMSLoader":
-        issues: list[NeatIssue] = []
-        data_model: dm.DataModel[dm.View] | None = None
-        try:
-            data_model = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True).latest_version()
-        except Exception as e:
-            issues.append(ResourceRetrievalError(data_model_id, "data model", str(e)))
-
-        return cls(graph_store, data_model, instance_space, {}, issues, client=client)
-
-    @classmethod
-    def from_rules(
-        cls,
-        rules: DMSRules,
-        graph_store: NeatGraphStore,
-        instance_space: str,
-        client: NeatClient | None = None,
-        unquote_external_ids: bool = False,
-    ) -> "DMSLoader":
-        issues: list[NeatIssue] = []
-        data_model: dm.DataModel[dm.View] | None = None
-        try:
-            data_model = rules.as_schema().as_read_model()
-        except Exception as e:
-            issues.append(
-                ResourceConversionError(
-                    identifier=rules.metadata.as_identifier(),
-                    resource_type="DMS Rules",
-                    target_format="read DMS model",
-                    reason=str(e),
-                )
-            )
-
-        class_neat_id_by_view_id = {view.view.as_id(): view.logical for view in rules.views if view.logical}
-
-        return cls(
-            graph_store,
-            data_model,
-            instance_space,
-            class_neat_id_by_view_id,
-            issues,
-            rules=rules,
-            client=client,
-            unquote_external_ids=unquote_external_ids,
-        )
-
-    def _load(self, stop_on_exception: bool = False) -> Iterable[dm.InstanceApply | NeatIssue | type[_END_OF_CLASS]]:
-        if self._issues.has_errors and stop_on_exception:
-            raise self._issues.as_exception()
-        elif self._issues.has_errors:
-            yield from self._issues
-            return
-        if not self.data_model:
-            # There should already be an error in this case.
-            return
-
-        views_with_linked_properties = (
-            DMSAnalysis(self.rules).views_with_properties_linked_to_classes(consider_inheritance=True)
-            if self.rules and self.rules.metadata.logical
-            else None
-        )
-
-        view_and_count_by_id = self._select_views_with_instances(self.data_model.views)
-
-        if self._client:
-            view_and_count_by_id, properties_point_to_self = self._sort_by_direct_relation_dependencies(
-                view_and_count_by_id
-            )
-        else:
-            properties_point_to_self = {}
-
-        view_ids: list[str] = []
-        for view_id in view_and_count_by_id.keys():
-            view_ids.append(repr(view_id))
-            if view_id in properties_point_to_self:
-                # If the views have a dependency on themselves, we need to run it twice.
-                view_ids.append(f"{view_id!r} (self)")
-
-        tracker = self._tracker(type(self).__name__, view_ids, "views")
-        for view_id, (view, instance_count) in view_and_count_by_id.items():
-            pydantic_cls, edge_by_type, edge_by_prop_id, issues = self._create_validation_classes(view)  # type: ignore[var-annotated]
-            yield from issues
-            tracker.issue(issues)
-
-            if view_id in properties_point_to_self:
-                # If the view has a dependency on itself, we need to run it twice.
-                # First, to ensure that all nodes are created, and then to add the direct relations.
-                # This only applies if there is a require constraint on the container, if not
-                # we can create an empty node on the fly.
-                iterations = [properties_point_to_self[view_id], set()]
-            else:
-                iterations = [set()]
-
-            for skip_properties in iterations:
-                if skip_properties:
-                    track_id = f"{view_id} (self)"
-                else:
-                    track_id = repr(view_id)
-                tracker.start(track_id)
-                if views_with_linked_properties:
-                    # we need graceful exit if the view is not in the view_property_pairs
-                    property_link_pairs = views_with_linked_properties.get(ViewEntity.from_id(view_id))
-
-                    if class_neat_id := self.class_neat_id_by_view_id.get(view_id):
-                        reader = self.graph_store._read_via_rules_linkage(class_neat_id, property_link_pairs)
-                    else:
-                        error_view = ResourceRetrievalError(view_id, "view", "View not linked to class")
-                        tracker.issue(error_view)
-                        if stop_on_exception:
-                            raise error_view
-                        yield error_view
-                        continue
-                else:
-                    # this assumes no changes in the suffix of view and class
-                    reader = self.graph_store.read(view.external_id)
-
-                instance_iterable = iterate_progress_bar_if_above_config_threshold(
-                    reader, instance_count, f"Loading {track_id}"
-                )
-
-                for identifier, properties in instance_iterable:
-                    start_node, end_node = self._pop_start_end_node(properties)
-                    is_edge = start_node and end_node
-                    if (is_edge and view.used_for == "node") or (not is_edge and view.used_for == "edge"):
-                        instance_type = "edge" if is_edge else "node"
-                        creation_error = ResourceCreationError(
-                            identifier,
-                            instance_type,
-                            error=f"{instance_type.capitalize()} found in {view.used_for} view",
-                        )
-                        tracker.issue(creation_error)
-                        if stop_on_exception:
-                            raise creation_error
-                        yield creation_error
-                        continue
-
-                    if skip_properties:
-                        properties = {k: v for k, v in properties.items() if k not in skip_properties}
-
-                    if start_node and end_node:
-                        # Is an edge
-                        try:
-                            yield self._create_edge_with_properties(
-                                identifier, properties, start_node, end_node, pydantic_cls, view_id
-                            )
-                        except ValueError as e:
-                            error_edge = ResourceCreationError(identifier, "edge", error=str(e))
-                            tracker.issue(error_edge)
-                            if stop_on_exception:
-                                raise error_edge from e
-                            yield error_edge
-                    else:
-                        try:
-                            yield self._create_node(identifier, properties, pydantic_cls, view_id)
-                        except ValueError as e:
-                            error_node = ResourceCreationError(identifier, "node", error=str(e))
-                            tracker.issue(error_node)
-                            if stop_on_exception:
-                                raise error_node from e
-                            yield error_node
-                        yield from self._create_edges_without_properties(
-                            identifier, properties, edge_by_type, edge_by_prop_id, tracker
-                        )
-                tracker.finish(track_id)
-                yield _END_OF_CLASS
-
-    @staticmethod
-    def _pop_start_end_node(properties: dict[str | InstanceType, list[str]]) -> tuple[str | None, str | None]:
-        start_node = properties.pop("startNode", [None])[0]
-        if not start_node:
-            start_node = properties.pop("start_node", [None])[0]
-        end_node = properties.pop("endNode", [None])[0]
-        if not end_node:
-            end_node = properties.pop("end_node", [None])[0]
-        return start_node, end_node
-
     def write_to_file(self, filepath: Path) -> None:
         if filepath.suffix not in [".json", ".yaml", ".yml"]:
             raise ValueError(f"File format {filepath.suffix} is not supported")
@@ -286,76 +148,203 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             else:
                 yaml.safe_dump(dumped, f, sort_keys=False)
 
-    def _select_views_with_instances(self, views: list[dm.View]) -> dict[dm.ViewId, tuple[dm.View, int]]:
-        """Selects the views with data."""
-        view_and_count_by_id: dict[dm.ViewId, tuple[dm.View, int]] = {}
-        uri_by_type: dict[str, URIRef] = {
-            remove_namespace_from_uri(uri[0]): uri[0]  # type: ignore[misc]
-            for uri in self.graph_store.queries.list_types()
-        }
-        for view in views:
-            view_id = view.as_id()
-            neat_id = self.class_neat_id_by_view_id.get(view_id)
-            if neat_id is not None:
-                count = self.graph_store.count_of_id(neat_id)
-            elif view.external_id in uri_by_type:
-                count = self.graph_store.count_of_type(uri_by_type[view.external_id])
-            else:
+    def _load(
+        self, stop_on_exception: bool = False
+    ) -> Iterable[dm.InstanceApply | NeatIssue | type[_END_OF_CLASS] | _START_OF_CLASS]:
+        if self._issues.has_errors and stop_on_exception:
+            raise self._issues.as_exception()
+        elif self._issues.has_errors:
+            yield from self._issues
+            return
+        view_iterations, issues = self._create_view_iterations()
+        yield from issues
+        if self._space_property:
+            yield from self._lookup_space_by_uri(view_iterations, stop_on_exception)
+
+            if self._client:
+                space_creation = self._create_instance_space_if_not_exists()
+                yield from space_creation.warnings
+                if space_creation.has_errors and stop_on_exception:
+                    raise space_creation.as_exception()
+                yield from space_creation.errors
+
+        if self.neat_prefix_by_type_uri:
+            self._lookup_identifier_by_uri()
+
+        for it in view_iterations:
+            view = it.view
+            if view is None:
+                yield ResourceNotFoundError(it.view_id, "view", more=f"Skipping {it.instance_count} instances...")
                 continue
-            if count > 0:
-                view_and_count_by_id[view_id] = view, count
+            projection, issues = self._create_projection(view)
+            yield from issues
+            query = it.query
+            reader = self.graph_store.read(
+                query.rdf_type,
+                property_renaming_config=query.property_renaming_config,
+                remove_uri_namespace=False,
+            )
+            instance_iterable = iterate_progress_bar_if_above_config_threshold(
+                reader, it.instance_count, f"Loading {it.view_id!r}"
+            )
+            yield _START_OF_CLASS(view.external_id)
+            for identifier, properties in instance_iterable:
+                yield from self._create_instances(
+                    identifier, properties, projection, stop_on_exception, exclude=it.hierarchical_properties
+                )
+            if it.hierarchical_properties:
+                # Force the creation of instances, before we create the hierarchical properties.
+                yield _END_OF_CLASS
+                yield _START_OF_CLASS(f"{view.external_id} ({humanize_collection(it.hierarchical_properties)})")
+                yield from self._create_hierarchical_properties(it, projection, stop_on_exception)
+            if reader is not instance_iterable:
+                print(f"Loaded {it.instance_count} instances for {it.view_id!r}")
+
+            yield _END_OF_CLASS
+
+    def _create_hierarchical_properties(
+        self, it: _ViewIterator, projection: _Projection, stop_on_exception: bool
+    ) -> Iterable[dm.InstanceApply | NeatIssue]:
+        reader = self.graph_store.read(it.query.rdf_type, property_renaming_config=it.query.property_renaming_config)
+        instance_iterable = iterate_progress_bar_if_above_config_threshold(
+            reader,
+            it.instance_count,
+            f"Loading {it.view_id!r} hierarchical properties: {humanize_collection(it.hierarchical_properties)}",
+        )
+        for identifier, properties in instance_iterable:
+            yield from self._create_instances(
+                identifier, properties, projection, stop_on_exception, include=it.hierarchical_properties
+            )
 
-        return view_and_count_by_id
+    def _create_view_iterations(self) -> tuple[list[_ViewIterator], IssueList]:
+        view_query_by_id = RulesAnalysis(self.info_rules, self.dms_rules).view_query_by_id
+        iterations_by_view_id = self._select_views_with_instances(view_query_by_id)
+        if self._client:
+            issues = IssueList()
+            views = self._client.data_modeling.views.retrieve(
+                list(iterations_by_view_id.keys()), include_inherited_properties=True
+            )
+            if missing := set(iterations_by_view_id) - {view.as_id() for view in views}:
+                for missing_view in missing:
+                    issues.append(ResourceNotFoundError(missing_view, "view", more="The view is not found in CDF."))
+                return [], issues
+            # Todo: Remove if this turns out to be unnecessary.
+            hierarchical_properties_by_view_id: dict[dm.ViewId, set[str]] = {}
+        else:
+            views = dm.ViewList([])
+            with catch_issues() as issues:
+                read_model = self.dms_rules.as_schema().as_read_model()
+                views.extend(read_model.views)
+            if issues.has_errors:
+                return [], issues
+            hierarchical_properties_by_view_id = {}
+        views_by_id = {view.as_id(): view for view in views}
+
+        def sort_by_instance_type(id_: dm.ViewId) -> int:
+            if id_ not in views_by_id:
+                return 0
+            return {"node": 1, "all": 1, "edge": 3}.get(views_by_id[id_].used_for, 0)
+
+        ordered_view_ids = SchemaAPI.get_view_order_by_direct_relation_constraints(views)
+        # Sort is stable in Python, so we will keep the order of the views:
+        ordered_view_ids.sort(key=sort_by_instance_type)
+        view_iterations: list[_ViewIterator] = []
+        for view_id in ordered_view_ids:
+            if view_id not in iterations_by_view_id:
+                continue
+            view_iteration = iterations_by_view_id[view_id]
+            view_iteration.view = views_by_id.get(view_id)
+            view_iteration.hierarchical_properties = hierarchical_properties_by_view_id.get(view_id, set())
+            view_iterations.append(view_iteration)
+        return view_iterations, issues
 
-    def _sort_by_direct_relation_dependencies(
-        self, view_and_count_by_id: dict[dm.ViewId, tuple[dm.View, int]]
-    ) -> tuple[dict[dm.ViewId, tuple[dm.View, int]], dict[dm.ViewId, set[str]]]:
-        """Sorts the views by container constraints."""
-        if not self._client:
-            return view_and_count_by_id, {}
-        # We need to retrieve the views to ensure we get all properties, such that we can find all
-        # the containers that the view is linked to.
-        views = self._client.data_modeling.views.retrieve(
-            list(view_and_count_by_id.keys()), include_inherited_properties=True
+    def _select_views_with_instances(self, view_query_by_id: ViewQueryDict) -> dict[dm.ViewId, _ViewIterator]:
+        """Selects the views with data."""
+        view_iterations: dict[dm.ViewId, _ViewIterator] = {}
+        for view_id, query in view_query_by_id.items():
+            count = self.graph_store.queries.count_of_type(query.rdf_type)
+            if count > 0:
+                view_iterations[view_id] = _ViewIterator(view_id, count, set(), query)
+        return view_iterations
+
+    def _lookup_space_by_uri(self, view_iterations: list[_ViewIterator], stop_on_exception: bool = False) -> IssueList:
+        issues = IssueList()
+        if self._space_property is None:
+            return issues
+        total = sum(it.instance_count for it in view_iterations)
+        properties_by_uriref = self.graph_store.queries.properties()
+        space_property_uri = next((k for k, v in properties_by_uriref.items() if v == self._space_property), None)
+        if space_property_uri is None:
+            error: ResourceNotFoundError[str, str] = ResourceNotFoundError(
+                self._space_property,
+                "property",
+                more=f"Could not find the {self._space_property} in the graph.",
+            )
+            if stop_on_exception:
+                raise error
+            issues.append(error)
+            return issues
+
+        instance_iterable = self.graph_store.queries.list_instances_ids_by_space(space_property_uri)
+        instance_iterable = iterate_progress_bar_if_above_config_threshold(
+            instance_iterable, total, f"Looking up spaces for {total} instances..."
         )
-        container_ids_by_view_id = {view.as_id(): view.referenced_containers() for view in views}
-        referenced_containers = {
-            container for containers in container_ids_by_view_id.values() for container in containers
-        }
-        containers = self._client.data_modeling.containers.retrieve(list(referenced_containers))
-        container_by_id = {container.as_id(): container for container in containers}
-
-        dependency_on_self: dict[dm.ViewId, set[str]] = defaultdict(set)
-        view_id_by_dependencies: dict[dm.ViewId, set[dm.ViewId]] = {}
-        for view in views:
-            view_id = view.as_id()
-            dependencies = set()
-            for prop_id, prop in view.properties.items():
-                if isinstance(prop, dm.MappedProperty) and isinstance(prop.type, dm.DirectRelation) and prop.source:
-                    container = container_by_id[prop.container]
-                    has_require_constraint = any(
-                        isinstance(constraint, dm.RequiresConstraint) for constraint in container.constraints.values()
+        neat_prefix = self.neat_prefix_by_predicate_uri.get(space_property_uri)
+        warned_spaces: set[str] = set()
+        for instance, space in instance_iterable:
+            if neat_prefix:
+                space = space.removeprefix(neat_prefix)
+
+            clean_space = NamingStandardization.standardize_space_str(space)
+            if clean_space != space and space not in warned_spaces:
+                issues.append(
+                    NeatValueWarning(
+                        f"Invalid space in property {self._space_property}: {space}. Fixed to {clean_space}"
                     )
-                    if has_require_constraint and prop.source == view_id:
-                        dependency_on_self[view_id].add(prop_id)
-                    elif has_require_constraint:
-                        dependencies.add(prop.source)
-            view_id_by_dependencies[view_id] = dependencies
-
-        ordered_view_ids = TopologicalSorter(view_id_by_dependencies).static_order()
-
-        return {
-            view_id: view_and_count_by_id[view_id] for view_id in ordered_view_ids if view_id in view_and_count_by_id
-        }, dict(dependency_on_self)
-
-    def _create_validation_classes(
-        self, view: dm.View
-    ) -> tuple[
-        type[BaseModel],
-        dict[str, tuple[str, dm.EdgeConnection]],
-        dict[str, tuple[str, dm.EdgeConnection]],
-        NeatIssueList,
-    ]:
+                )
+                warned_spaces.add(space)
+
+            self._space_by_instance_uri[instance] = clean_space
+        return issues
+
+    def _lookup_identifier_by_uri(self) -> None:
+        if not self.neat_prefix_by_type_uri:
+            return
+
+        count = sum(count for _, count in self.graph_store.queries.summarize_instances())
+        instance_iterable = self.graph_store.queries.list_instances_ids()
+        instance_iterable = iterate_progress_bar_if_above_config_threshold(
+            instance_iterable, count, f"Looking up identifiers for {count} instances..."
+        )
+        count_by_identifier: dict[str, list[URIRef]] = defaultdict(list)
+        for instance_uri, type in instance_iterable:
+            if type not in self.neat_prefix_by_type_uri:
+                continue
+            prefix = self.neat_prefix_by_type_uri[type]
+            identifier = remove_namespace_from_uri(instance_uri)
+            if self._unquote_external_ids:
+                identifier = urllib.parse.unquote(identifier)
+            count_by_identifier[identifier.removeprefix(prefix)].append(instance_uri)
+
+        for identifier, uris in count_by_identifier.items():
+            if len(uris) == 1:
+                self._external_id_by_uri[uris[0]] = identifier
+
+    def _create_instance_space_if_not_exists(self) -> IssueList:
+        issues = IssueList()
+        if not self._client:
+            return issues
+
+        instance_spaces = set(self._space_by_instance_uri.values()) - {self._instance_space}
+        existing_spaces = {space.space for space in self._client.data_modeling.spaces.retrieve(list(instance_spaces))}
+        if missing_spaces := (instance_spaces - existing_spaces):
+            try:
+                self._client.data_modeling.spaces.apply([dm.SpaceApply(space=space) for space in missing_spaces])
+            except CogniteAPIError as e:
+                issues.append(AuthorizationError(f"Creating {len(missing_spaces)} instance spaces.", str(e)))
+        return issues
+
+    def _create_projection(self, view: dm.View) -> tuple[_Projection, IssueList]:
         issues = IssueList()
         field_definitions: dict[str, tuple[type, Any]] = {}
         edge_by_type: dict[str, tuple[str, dm.EdgeConnection]] = {}
@@ -414,9 +403,15 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
         def parse_list(cls, value: Any, info: ValidationInfo) -> list[str]:
             if isinstance(value, list) and list.__name__ not in _get_field_value_types(cls, info):
-                if len(value) == 1:
-                    return value[0]
-                raise ValueError(f"Got multiple values for {info.field_name}: {value}")
+                if len(value) > 1:
+                    warnings.warn(
+                        # the identifier is unknown, it will be cest in the create_instances method
+                        PropertyMultipleValueWarning("", "property", str(info.field_name), value=str(value[0])),
+                        stacklevel=2,
+                    )
+                elif not value:
+                    return None  # type: ignore[return-value]
+                return value[0]
 
             return value
 
@@ -446,7 +441,9 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             def parse_direct_relation(cls, value: list, info: ValidationInfo) -> dict | list[dict]:
                 # We validate above that we only get one value for single direct relations.
                 if list.__name__ in _get_field_value_types(cls, info):
-                    result = [{"space": self.instance_space, "externalId": remove_namespace_from_uri(v)} for v in value]
+                    ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
+                    result = [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
+                    # Todo: Account for max_list_limit
                     if len(result) <= DMS_DIRECT_RELATION_LIST_LIMIT:
                         return result
                     warnings.warn(
@@ -462,7 +459,9 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     result.sort(key=lambda x: (x["space"], x["externalId"]))
                     return result[:DMS_DIRECT_RELATION_LIST_LIMIT]
                 elif value:
-                    return {"space": self.instance_space, "externalId": remove_namespace_from_uri(value[0])}
+                    return self._create_instance_id(value[0], "node", stop_on_exception=True).dump(
+                        camel_case=True, include_instance_type=False
+                    )
                 return {}
 
             validators["parse_direct_relation"] = field_validator(*direct_relation_by_property.keys(), mode="before")(  # type: ignore[assignment]
@@ -473,78 +472,124 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
             def parse_direct_relation_to_unit(cls, value: Any, info: ValidationInfo) -> dict | list[dict]:
                 if value:
-                    return {"space": "cdf_cdm_units", "externalId": remove_namespace_from_uri(value[0])}
+                    external_id = remove_namespace_from_uri(value[0])
+                    if self._unquote_external_ids:
+                        external_id = urllib.parse.unquote(external_id)
+                    return {"space": "cdf_cdm_units", "externalId": external_id}
                 return {}
 
             validators["parse_direct_relation_to_unit"] = field_validator(*unit_properties, mode="before")(  # type: ignore[assignment]
                 parse_direct_relation_to_unit  # type: ignore[arg-type]
             )
 
+        if text_fields:
+
+            def parse_text(cls, value: Any, info: ValidationInfo) -> str | list[str]:
+                if isinstance(value, list):
+                    return [remove_namespace_from_uri(v) if isinstance(v, URIRef) else str(v) for v in value]
+                return remove_namespace_from_uri(value) if isinstance(value, URIRef) else str(value)
+
+            validators["parse_text"] = field_validator(*text_fields, mode="before")(parse_text)  # type: ignore[assignment, arg-type]
+
         pydantic_cls = create_model(view.external_id, __validators__=validators, **field_definitions)  # type: ignore[arg-type, call-overload]
-        return pydantic_cls, edge_by_type, edge_by_prop_id, issues
+        return _Projection(view.as_id(), view.used_for, pydantic_cls, edge_by_type, edge_by_prop_id), issues
 
-    def _create_node(
+    def _create_instances(
         self,
-        identifier: str,
-        properties: dict[str | InstanceType, list[str]],
-        pydantic_cls: type[BaseModel],
-        view_id: dm.ViewId,
-    ) -> dm.InstanceApply:
-        type_ = properties.pop(RDF.type, [None])[0]
-        created = pydantic_cls.model_validate(properties)
-        if self._unquote_external_ids:
-            identifier = urllib.parse.unquote(identifier)
-
-        return dm.NodeApply(
-            space=self.instance_space,
-            external_id=identifier,
-            type=(dm.DirectRelationReference(view_id.space, view_id.external_id) if type_ is not None else None),
-            sources=[
-                dm.NodeOrEdgeData(source=view_id, properties=dict(created.model_dump(exclude_unset=True).items()))
-            ],
-        )
+        instance_uri: URIRef,
+        properties: dict[str | InstanceType, list[Any]],
+        projection: _Projection,
+        stop_on_exception: Literal[True, False] = False,
+        exclude: set[str] | None = None,
+        include: set[str] | None = None,
+    ) -> Iterable[dm.InstanceApply | NeatIssue]:
+        instance_id = self._create_instance_id(instance_uri, "node", stop_on_exception)
+        if not isinstance(instance_id, InstanceId):
+            yield instance_id
+            return
+        space, external_id = instance_id.space, instance_id.external_id
+        start_node, end_node = self._pop_start_end_node(properties)
+        is_edge = start_node and end_node
+        instance_type = "edge" if is_edge else "node"
+        if (projection.used_for == "node" and is_edge) or (projection.used_for == "edge" and not is_edge):
+            creation_error = ResourceCreationError(
+                external_id,
+                instance_type,
+                f"View used for {projection.used_for} instance {external_id!s} but is {instance_type}",
+            )
+            if stop_on_exception:
+                raise creation_error from None
+            yield creation_error
+            return
 
-    def _create_edge_with_properties(
-        self,
-        identifier: str,
-        properties: dict[str | InstanceType, list[str]],
-        start_node: str,
-        end_node: str,
-        pydantic_cls: type[BaseModel],
-        view_id: dm.ViewId,
-    ) -> dm.EdgeApply:
-        type_ = properties.pop(RDF.type, [None])[0]
-        created = pydantic_cls.model_validate(properties)
-        if type_ is None:
-            raise ValueError(f"Missing type for edge {identifier}")
-
-        if self._unquote_external_ids:
-            identifier = urllib.parse.unquote(identifier)
-
-        return dm.EdgeApply(
-            space=self.instance_space,
-            external_id=identifier,
-            type=dm.DirectRelationReference(view_id.space, view_id.external_id),
-            start_node=dm.DirectRelationReference(self.instance_space, start_node),
-            end_node=dm.DirectRelationReference(self.instance_space, end_node),
-            sources=[
-                dm.NodeOrEdgeData(source=view_id, properties=dict(created.model_dump(exclude_unset=True).items()))
-            ],
-        )
+        if RDF.type not in properties:
+            error = ResourceCreationError(external_id, instance_type, "No rdf:type found")
+            if stop_on_exception:
+                raise error from None
+            yield error
+            return
+        _ = properties.pop(RDF.type)[0]
+        if exclude:
+            properties = {k: v for k, v in properties.items() if k not in exclude}
+        if include:
+            properties = {k: v for k, v in properties.items() if k in include}
+
+        sources = []
+        with catch_issues() as property_issues:
+            sources = [
+                dm.NodeOrEdgeData(
+                    projection.view_id,
+                    projection.pydantic_cls.model_validate(properties).model_dump(exclude_unset=True),
+                )
+            ]
+        for issue in property_issues:
+            if isinstance(issue, ResourceNeatWarning):
+                issue.identifier = external_id
+
+        if property_issues.has_errors and stop_on_exception:
+            raise property_issues.as_exception()
+        yield from property_issues
+        if not sources:
+            return
+
+        if start_node and end_node:
+            start = self._create_instance_id(start_node, "edge", stop_on_exception)
+            end = self._create_instance_id(end_node, "edge", stop_on_exception)
+            if isinstance(start, NeatError):
+                yield start
+            if isinstance(end, NeatError):
+                yield end
+            if isinstance(start, InstanceId) and isinstance(end, InstanceId):
+                yield dm.EdgeApply(
+                    space=space,
+                    external_id=external_id,
+                    type=(projection.view_id.space, projection.view_id.external_id),
+                    start_node=start.as_tuple(),
+                    end_node=end.as_tuple(),
+                    sources=sources,
+                )
+        else:
+            yield dm.NodeApply(
+                space=space,
+                external_id=external_id,
+                type=(projection.view_id.space, projection.view_id.external_id),
+                sources=sources,
+            )
+        yield from self._create_edges_without_properties(space, external_id, properties, projection, stop_on_exception)
 
     def _create_edges_without_properties(
         self,
+        space: str,
         identifier: str,
-        properties: dict[str, list[str]],
-        edge_by_type: dict[str, tuple[str, dm.EdgeConnection]],
-        edge_by_prop_id: dict[str, tuple[str, dm.EdgeConnection]],
-        tracker: Tracker,
+        properties: dict[str | InstanceType, list[str] | list[URIRef]],
+        projection: _Projection,
+        stop_on_exception: Literal[True, False],
     ) -> Iterable[dm.EdgeApply | NeatIssue]:
         for predicate, values in properties.items():
-            if predicate in edge_by_type:
-                prop_id, edge = edge_by_type[predicate]
-            elif predicate in edge_by_prop_id:
-                prop_id, edge = edge_by_prop_id[predicate]
+            if predicate in projection.edge_by_type:
+                prop_id, edge = projection.edge_by_type[predicate]
+            elif predicate in projection.edge_by_prop_id:
+                prop_id, edge = projection.edge_by_prop_id[predicate]
             else:
                 continue
             if isinstance(edge, SingleEdgeConnection) and len(values) > 1:
@@ -553,21 +598,83 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     identifier=identifier,
                     location=f"Multiple values for single edge {edge}. Expected only one.",
                 )
-                tracker.issue(error)
                 yield error
+                continue
             for target in values:
+                target_id = self._create_instance_id(target, "edge", stop_on_exception)  # type: ignore[call-overload]
+                if not isinstance(target_id, InstanceId):
+                    yield target_id
+                    continue
+                if isinstance(target, URIRef):
+                    target = remove_namespace_from_uri(target)
                 external_id = f"{identifier}.{prop_id}.{target}"
-                if self._unquote_external_ids:
-                    external_id = urllib.parse.unquote(external_id)
 
+                start_node, end_node = (
+                    (space, identifier),
+                    target_id.as_tuple(),
+                )
+                if edge.direction == "inwards":
+                    start_node, end_node = end_node, start_node
                 yield dm.EdgeApply(
-                    space=self.instance_space,
+                    space=space,
                     external_id=(external_id if len(external_id) < 256 else create_sha256_hash(external_id)),
                     type=edge.type,
-                    start_node=dm.DirectRelationReference(self.instance_space, identifier),
-                    end_node=dm.DirectRelationReference(self.instance_space, target),
+                    start_node=start_node,
+                    end_node=end_node,
                 )
 
+    @staticmethod
+    def _pop_start_end_node(
+        properties: dict[str | InstanceType, list[str] | list[URIRef]],
+    ) -> tuple[URIRef, URIRef] | tuple[None, None]:
+        start_node = properties.pop("startNode", [None])[0]
+        if not start_node:
+            start_node = properties.pop("start_node", [None])[0]
+        end_node = properties.pop("endNode", [None])[0]
+        if not end_node:
+            end_node = properties.pop("end_node", [None])[0]
+        if start_node and end_node:
+            return start_node, end_node  # type: ignore[return-value]
+        return None, None
+
+    @overload
+    def _create_instance_id(
+        self, uri: URIRef, instance_type: str, stop_on_exception: Literal[False] = False
+    ) -> InstanceId | NeatError: ...
+
+    @overload
+    def _create_instance_id(
+        self, uri: URIRef, instance_type: str, stop_on_exception: Literal[True] = True
+    ) -> InstanceId: ...
+
+    def _create_instance_id(
+        self, uri: URIRef, instance_type: str, stop_on_exception: bool = False
+    ) -> InstanceId | NeatError:
+        space: str | None = None
+        external_id: str | None = None
+        error: NeatError | None = None
+        if self._use_source_space:
+            namespace, external_id = split_uri(uri)
+            space = namespace_as_space(namespace)
+            if space is None:
+                error = ResourceCreationError(uri, instance_type, f"Could not find space for {uri!s}.")
+        else:
+            space = self._space_by_instance_uri[uri]
+            if uri in self._external_id_by_uri:
+                external_id = self._external_id_by_uri[uri]
+            else:
+                external_id = remove_namespace_from_uri(uri)
+
+        if external_id and self._unquote_external_ids:
+            external_id = urllib.parse.unquote(external_id)
+        if space and external_id:
+            return InstanceId(space, external_id)
+        if error is None:
+            raise ValueError(f"Bug in neat. Failed to create instance ID and determine error for {uri!r}")
+        if stop_on_exception:
+            raise error
+        return error
+
     def _get_required_capabilities(self) -> list[Capability]:
         return [
             DataModelInstancesAcl(
@@ -576,7 +683,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     DataModelInstancesAcl.Action.Write_Properties,
                     DataModelInstancesAcl.Action.Read,
                 ],
-                scope=DataModelInstancesAcl.Scope.SpaceID([self.instance_space]),
+                scope=DataModelInstancesAcl.Scope.SpaceID([self._instance_space]),
             )
         ]
 
@@ -586,24 +693,11 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         items: list[dm.InstanceApply],
         dry_run: bool,
         read_issues: IssueList,
+        class_name: str | None = None,
     ) -> Iterable[UploadResult]:
-        nodes: list[dm.NodeApply] = []
-        edges: list[dm.EdgeApply] = []
-        source_by_node_id: dict[dm.NodeId, str] = {}
-        source_by_edge_id: dict[dm.EdgeId, str] = {}
-        for item in items:
-            if isinstance(item, dm.NodeApply):
-                nodes.append(item)
-                if item.sources:
-                    source_by_node_id[item.as_id()] = item.sources[0].source.external_id
-                else:
-                    source_by_node_id[item.as_id()] = "node"
-            elif isinstance(item, dm.EdgeApply):
-                edges.append(item)
-                if item.sources:
-                    source_by_edge_id[item.as_id()] = item.sources[0].source.external_id
-                else:
-                    source_by_edge_id[item.as_id()] = "edge"
+        name = class_name or "Instances"
+        nodes = [item for item in items if isinstance(item, dm.NodeApply)]
+        edges = [item for item in items if isinstance(item, dm.EdgeApply)]
         try:
             upserted = client.data_modeling.instances.apply(
                 nodes,
@@ -613,29 +707,28 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 skip_on_version_conflict=True,
             )
         except CogniteAPIError as e:
-            result = UploadResult[InstanceId](name="Instances", issues=read_issues)
-            result.error_messages.append(str(e))
-            result.failed_upserted.update(item.as_id() for item in e.failed + e.unknown)
-            result.created.update(item.as_id() for item in e.successful)
-            yield result
+            if len(items) == 1:
+                yield UploadResult(
+                    name=name,
+                    issues=read_issues,
+                    failed_items=items,
+                    error_messages=[str(e)],
+                    failed_upserted={item.as_id() for item in items},  # type: ignore[attr-defined]
+                )
+            else:
+                half = len(items) // 2
+                yield from self._upload_to_cdf(client, items[:half], dry_run, read_issues, class_name)
+                yield from self._upload_to_cdf(client, items[half:], dry_run, read_issues, class_name)
         else:
-            for instances, ids_by_source in [
-                (upserted.nodes, source_by_node_id),
-                (upserted.edges, source_by_edge_id),
-            ]:
-                for name, subinstances in itertools.groupby(
-                    sorted(instances, key=lambda i: ids_by_source.get(i.as_id(), "")),  # type: ignore[call-overload, index, attr-defined]
-                    key=lambda i: ids_by_source.get(i.as_id(), ""),  # type: ignore[index, attr-defined]
-                ):
-                    result = UploadResult(name=name, issues=read_issues)
-                    for instance in subinstances:  # type: ignore[attr-defined]
-                        if instance.was_modified and instance.created_time == instance.last_updated_time:
-                            result.created.add(instance.as_id())
-                        elif instance.was_modified:
-                            result.changed.add(instance.as_id())
-                        else:
-                            result.unchanged.add(instance.as_id())
-                    yield result
+            result = UploadResult(name=name, issues=read_issues)  # type: ignore[var-annotated]
+            for instance in itertools.chain(upserted.nodes, upserted.edges):  # type: ignore[attr-defined]
+                if instance.was_modified and instance.created_time == instance.last_updated_time:
+                    result.created.add(instance.as_id())
+                elif instance.was_modified:
+                    result.changed.add(instance.as_id())
+                else:
+                    result.unchanged.add(instance.as_id())
+            yield result
 
 
 def _get_field_value_types(cls, info):