cognite-neat 0.122.0__tar.gz → 0.122.2__tar.gz

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.122.0
+Version: 0.122.2
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
@@ -47,9 +47,9 @@ Requires-Dist: google-auth-oauthlib<2.0.0,>=1.0.0; extra == 'google'
 Requires-Dist: gspread<6.0.0,>=5.0.0; extra == 'google'
 Provides-Extra: lxml
 Requires-Dist: lxml<6.0.0,>=5.3.0; extra == 'lxml'
-Provides-Extra: pyoxigraph
-Requires-Dist: oxrdflib<0.5.0,>=0.4.0; extra == 'pyoxigraph'
-Requires-Dist: pyoxigraph<0.5.0,>=0.4.3; extra == 'pyoxigraph'
+Provides-Extra: oxi
+Requires-Dist: oxrdflib<0.5.0,>=0.4.0; extra == 'oxi'
+Requires-Dist: pyoxigraph<0.5.0,>=0.4.3; extra == 'oxi'
 Description-Content-Type: text/markdown
 
 # kNowlEdge grAph Transformer (NEAT)

cognite_neat-0.122.2/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.122.2"
+__engine__ = "^2.0.4"

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_data_model/importers/_dms2data_model.py

@@ -269,14 +269,14 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
 
         model = self.schema.data_model
 
-        user_data_model = self._create_rule_components(model, self.schema, self.metadata)
+        user_data_model = self._create_data_model_components(model, self.schema, self.metadata)
 
         self.issue_list.trigger_warnings()
         if self.issue_list.has_errors:
             raise MultiValueError(self.issue_list.errors)
         return ImportedDataModel(user_data_model, {})
 
-    def _create_rule_components(
+    def _create_data_model_components(
         self,
         data_model: dm.DataModelApply,
         schema: DMSSchema,

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_data_model/transformers/_converters.py

@@ -2346,7 +2346,7 @@ class AddCogniteProperties(
     ) -> ImportedDataModel[UnverifiedConceptualDataModel]:
         input_ = data_model.unverified_data_model
         if input_ is None:
-            raise NeatValueError("Rule read failed. Cannot add cognite properties to None data_model.")
+            raise NeatValueError("Data model read failed. Cannot add cognite properties to None data_model.")
 
         default_space = input_.metadata.space
         default_version = input_.metadata.version

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_data_model/transformers/_mapping.py

@@ -29,7 +29,7 @@ from ._base import VerifiedDataModelTransformer
 
 
 class MapOntoTransformers(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
-    """Base class for transformers that map one rule onto another."""
+    """Base class for transformers that map one data model onto another."""
 
     ...
 

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_data_model/transformers/_verification.py

@@ -54,7 +54,7 @@ class VerificationTransformer(DataModelTransformer[T_ImportedUnverifiedDataModel
                 elif issubclass(validation_cls, ConceptualValidation):
                     validation_issues = ConceptualValidation(verified_data_model, data_model.context).validate()  # type: ignore[arg-type]
                 else:
-                    raise NeatValueError("Unsupported rule type")
+                    raise NeatValueError("Unsupported data model type")
                 issues.extend(validation_issues)
 
         # Raise issues which is expected to be handled outside of this method

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_instances/loaders/__init__.py

@@ -1,7 +1,8 @@
 from ._base import BaseLoader, CDFLoader
 from ._rdf2dms import DMSLoader
+from ._rdf_to_instance_space import InstanceSpaceLoader
 
-__all__ = ["BaseLoader", "CDFLoader", "DMSLoader"]
+__all__ = ["BaseLoader", "CDFLoader", "DMSLoader", "InstanceSpaceLoader"]
 
 
 def _repr_html_() -> str:

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_instances/loaders/_base.py

@@ -3,12 +3,11 @@ from collections.abc import Hashable, Iterable
 from pathlib import Path
 from typing import ClassVar, Generic, TypeVar
 
-from cognite.client import CogniteClient
 from cognite.client.data_classes.capabilities import Capability
 
+from cognite.neat.core._client import NeatClient
 from cognite.neat.core._issues import IssueList, NeatIssue
 from cognite.neat.core._issues.errors import AuthorizationError
-from cognite.neat.core._store import NeatInstanceStore
 from cognite.neat.core._utils.auxiliary import class_html_doc
 from cognite.neat.core._utils.upload import UploadResult, UploadResultList
 
@@ -29,9 +28,6 @@ class BaseLoader(ABC, Generic[T_Output]):
     _new_line = "\n"
     _encoding = "utf-8"
 
-    def __init__(self, instance_store: NeatInstanceStore):
-        self.instance_store = instance_store
-
     @abstractmethod
     def write_to_file(self, filepath: Path) -> None:
         raise NotImplementedError
@@ -59,9 +55,7 @@ class BaseLoader(ABC, Generic[T_Output]):
 class CDFLoader(BaseLoader[T_Output]):
     _UPLOAD_BATCH_SIZE: ClassVar[int] = 1000
 
-    def load_into_cdf(
-        self, client: CogniteClient, dry_run: bool = False, check_client: bool = True
-    ) -> UploadResultList:
+    def load_into_cdf(self, client: NeatClient, dry_run: bool = False, check_client: bool = True) -> UploadResultList:
         upload_result_by_name: dict[str, UploadResult] = {}
         for upload_result in self.load_into_cdf_iterable(client, dry_run, check_client):
             if last_result := upload_result_by_name.get(upload_result.name):
@@ -72,7 +66,7 @@ class CDFLoader(BaseLoader[T_Output]):
         return UploadResultList(upload_result_by_name.values())
 
     def load_into_cdf_iterable(
-        self, client: CogniteClient, dry_run: bool = False, check_client: bool = True
+        self, client: NeatClient, dry_run: bool = False, check_client: bool = True
     ) -> Iterable[UploadResult]:
         if check_client:
             missing_capabilities = client.iam.verify_capabilities(self._get_required_capabilities())
@@ -113,7 +107,7 @@ class CDFLoader(BaseLoader[T_Output]):
     @abstractmethod
     def _upload_to_cdf(
         self,
-        client: CogniteClient,
+        client: NeatClient,
         items: list[T_Output],
         dry_run: bool,
         read_issues: IssueList,

{cognite_neat-0.122.0 → cognite_neat-0.122.2}/cognite/neat/core/_instances/loaders/_rdf2dms.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Literal, cast, get_args, overload
+from typing import Any, Literal, cast, get_args
 
 import yaml
 from cognite.client import CogniteClient
@@ -22,6 +22,7 @@ from rdflib import RDF, URIRef
 from cognite.neat.core._client import NeatClient
 from cognite.neat.core._client._api_client import SchemaAPI
 from cognite.neat.core._constants import (
+    COGNITE_SPACES,
     DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
     is_readonly_property,
 )
@@ -38,13 +39,11 @@ from cognite.neat.core._data_model.models.data_types import (
 )
 from cognite.neat.core._issues import IssueList, NeatError, NeatIssue, catch_issues
 from cognite.neat.core._issues.errors import (
-    AuthorizationError,
     ResourceCreationError,
     ResourceDuplicatedError,
     ResourceNotFoundError,
 )
 from cognite.neat.core._issues.warnings import (
-    NeatValueWarning,
     PropertyDirectRelationLimitWarning,
     PropertyMultipleValueWarning,
     PropertyTypeNotSupportedWarning,
@@ -57,11 +56,8 @@ from cognite.neat.core._utils.collection_ import (
     iterate_progress_bar_if_above_config_threshold,
 )
 from cognite.neat.core._utils.rdf_ import (
-    namespace_as_space,
     remove_namespace_from_uri,
-    split_uri,
 )
-from cognite.neat.core._utils.text import NamingStandardization
 from cognite.neat.core._utils.upload import UploadResult
 
 from ._base import _END_OF_CLASS, _START_OF_CLASS, CDFLoader
@@ -103,14 +99,10 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         conceptual_data_model (ConceptualDataModel): Conceptual data model,
             used to look+up the instances in the store.
         instance_store (NeatInstanceStore): The instance store to load the instances from.
-        instance_space (str): The instance space to load the data into.
         create_issues (Sequence[NeatIssue] | None): A list of issues that occurred during reading. Defaults to None.
         client (NeatClient | None): This is used to lookup containers such that the loader
             creates instances in accordance with required constraints. Defaults to None.
         unquote_external_ids (bool): If True, the loader will unquote external ids before creating the instances.
-        neat_prefix_by_predicate_uri (dict[URIRef, str] | None): A dictionary that maps a predicate URIRef to a
-            prefix that Neat added to the object upon extraction. This is used to remove the prefix from the
-            object before creating the instance.
     """
 
     def __init__(
@@ -118,24 +110,17 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         physical_data_model: PhysicalDataModel,
         conceptual_data_model: ConceptualDataModel,
         instance_store: NeatInstanceStore,
-        instance_space: str,
-        space_property: str | None = None,
-        use_source_space: bool = False,
+        space_by_instance_uri: dict[URIRef, str],
         client: NeatClient | None = None,
         create_issues: Sequence[NeatIssue] | None = None,
         unquote_external_ids: bool = False,
-        neat_prefix_by_predicate_uri: dict[URIRef, str] | None = None,
         neat_prefix_by_type_uri: dict[URIRef, str] | None = None,
     ):
-        super().__init__(instance_store)
+        self.instance_store = instance_store
         self.physical_data_model = physical_data_model
         self.conceptual_data_model = conceptual_data_model
-        self.neat_prefix_by_predicate_uri = neat_prefix_by_predicate_uri or {}
         self.neat_prefix_by_type_uri = neat_prefix_by_type_uri or {}
-        self._instance_space = instance_space
-        self._space_property = space_property
-        self._use_source_space = use_source_space
-        self._space_by_instance_uri: dict[URIRef, str] = defaultdict(lambda: instance_space)
+        self._space_by_instance_uri = space_by_instance_uri
         self._external_id_by_uri: dict[URIRef, str] = {}
         self._issues = IssueList(create_issues or [])
         self._client = client
@@ -171,16 +156,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             return
         view_iterations, issues = self._create_view_iterations()
         yield from issues
-        if self._space_property:
-            yield from self._lookup_space_by_uri(view_iterations, stop_on_exception)
-
-            if self._client:
-                space_creation = self._create_instance_space_if_not_exists()
-                yield from space_creation.warnings
-                if space_creation.has_errors and stop_on_exception:
-                    raise space_creation.as_exception()
-                yield from space_creation.errors
-
         if self.neat_prefix_by_type_uri:
             self._lookup_identifier_by_uri()
 
@@ -279,46 +254,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 view_iterations[view_id] = _ViewIterator(view_id, count, query)
         return view_iterations
 
-    def _lookup_space_by_uri(self, view_iterations: list[_ViewIterator], stop_on_exception: bool = False) -> IssueList:
-        issues = IssueList()
-        if self._space_property is None:
-            return issues
-        total = sum(it.instance_count for it in view_iterations)
-        properties_by_uriref = self.instance_store.queries.select.properties()
-        space_property_uri = next((k for k, v in properties_by_uriref.items() if v == self._space_property), None)
-        if space_property_uri is None:
-            error: ResourceNotFoundError[str, str] = ResourceNotFoundError(
-                self._space_property,
-                "property",
-                more=f"Could not find the {self._space_property} in the graph.",
-            )
-            if stop_on_exception:
-                raise error
-            issues.append(error)
-            return issues
-
-        instance_iterable = self.instance_store.queries.select.list_instances_ids_by_space(space_property_uri)
-        instance_iterable = iterate_progress_bar_if_above_config_threshold(
-            instance_iterable, total, f"Looking up spaces for {total} instances..."
-        )
-        neat_prefix = self.neat_prefix_by_predicate_uri.get(space_property_uri)
-        warned_spaces: set[str] = set()
-        for instance, space in instance_iterable:
-            if neat_prefix:
-                space = space.removeprefix(neat_prefix)
-
-            clean_space = NamingStandardization.standardize_space_str(space)
-            if clean_space != space and space not in warned_spaces:
-                issues.append(
-                    NeatValueWarning(
-                        f"Invalid space in property {self._space_property}: {space}. Fixed to {clean_space}"
-                    )
-                )
-                warned_spaces.add(space)
-
-            self._space_by_instance_uri[instance] = clean_space
-        return issues
-
     def _lookup_identifier_by_uri(self) -> None:
         if not self.neat_prefix_by_type_uri:
             return
@@ -342,20 +277,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             if len(uris) == 1:
                 self._external_id_by_uri[uris[0]] = identifier
 
-    def _create_instance_space_if_not_exists(self) -> IssueList:
-        issues = IssueList()
-        if not self._client:
-            return issues
-
-        instance_spaces = set(self._space_by_instance_uri.values()) - {self._instance_space}
-        existing_spaces = {space.space for space in self._client.data_modeling.spaces.retrieve(list(instance_spaces))}
-        if missing_spaces := (instance_spaces - existing_spaces):
-            try:
-                self._client.data_modeling.spaces.apply([dm.SpaceApply(space=space) for space in missing_spaces])
-            except CogniteAPIError as e:
-                issues.append(AuthorizationError(f"Creating {len(missing_spaces)} instance spaces.", str(e)))
-        return issues
-
     def _create_projection(self, view: dm.View) -> tuple[_Projection, IssueList]:
         issues = IssueList()
         field_definitions: dict[str, tuple[type, Any]] = {}
@@ -472,12 +393,10 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                         )
                         value = value[:limit]
 
-                    ids = (self._create_instance_id(v, "node", stop_on_exception=True) for v in value)
+                    ids = (self._create_instance_id(v) for v in value)
                     return [id_.dump(camel_case=True, include_instance_type=False) for id_ in ids]
                 elif value:
-                    return self._create_instance_id(value[0], "node", stop_on_exception=True).dump(
-                        camel_case=True, include_instance_type=False
-                    )
+                    return self._create_instance_id(value[0]).dump(camel_case=True, include_instance_type=False)
                 return {}
 
             validators["parse_direct_relation"] = field_validator(*direct_relation_by_property.keys(), mode="before")(  # type: ignore[assignment]
@@ -517,7 +436,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         projection: _Projection,
         stop_on_exception: Literal[True, False] = False,
     ) -> Iterable[dm.InstanceApply | NeatIssue]:
-        instance_id = self._create_instance_id(instance_uri, "node", stop_on_exception)
+        instance_id = self._create_instance_id(instance_uri)
         if not isinstance(instance_id, InstanceId):
             yield instance_id
             return
@@ -566,8 +485,8 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             return
 
         if start_node and end_node:
-            start = self._create_instance_id(start_node, "edge", stop_on_exception)
-            end = self._create_instance_id(end_node, "edge", stop_on_exception)
+            start = self._create_instance_id(start_node)
+            end = self._create_instance_id(end_node)
             if isinstance(start, NeatError):
                 yield start
             if isinstance(end, NeatError):
@@ -585,10 +504,15 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             yield dm.NodeApply(
                 space=space,
                 external_id=external_id,
-                type=(projection.view_id.space, projection.view_id.external_id),
+                # Currently there are no node types for schemas in cognite schema spaces
+                type=(
+                    (projection.view_id.space, projection.view_id.external_id)
+                    if projection.view_id.space not in COGNITE_SPACES
+                    else None
+                ),
                 sources=sources,
             )
-        yield from self._create_edges_without_properties(space, external_id, properties, projection, stop_on_exception)
+        yield from self._create_edges_without_properties(space, external_id, properties, projection)
 
     def _create_edges_without_properties(
         self,
@@ -596,7 +520,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         identifier: str,
         properties: dict[str | InstanceType, list[str] | list[URIRef]],
         projection: _Projection,
-        stop_on_exception: Literal[True, False],
     ) -> Iterable[dm.EdgeApply | NeatIssue]:
         for predicate, values in properties.items():
             if predicate in projection.edge_by_type:
@@ -614,10 +537,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 yield error
                 continue
             for target in values:
-                target_id = self._create_instance_id(target, "edge", stop_on_exception)  # type: ignore[call-overload]
-                if not isinstance(target_id, InstanceId):
-                    yield target_id
-                    continue
+                target_id = self._create_instance_id(cast(URIRef, target))
                 if isinstance(target, URIRef):
                     target = remove_namespace_from_uri(target)
                 external_id = f"{identifier}.{prop_id}.{target}"
@@ -650,43 +570,16 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             return start_node, end_node  # type: ignore[return-value]
         return None, None
 
-    @overload
-    def _create_instance_id(
-        self, uri: URIRef, instance_type: str, stop_on_exception: Literal[False] = False
-    ) -> InstanceId | NeatError: ...
-
-    @overload
-    def _create_instance_id(
-        self, uri: URIRef, instance_type: str, stop_on_exception: Literal[True] = True
-    ) -> InstanceId: ...
-
-    def _create_instance_id(
-        self, uri: URIRef, instance_type: str, stop_on_exception: bool = False
-    ) -> InstanceId | NeatError:
-        space: str | None = None
-        external_id: str | None = None
-        error: NeatError | None = None
-        if self._use_source_space:
-            namespace, external_id = split_uri(uri)
-            space = namespace_as_space(namespace)
-            if space is None:
-                error = ResourceCreationError(uri, instance_type, f"Could not find space for {uri!s}.")
+    def _create_instance_id(self, uri: URIRef) -> InstanceId:
+        space = self._space_by_instance_uri[uri]
+        if uri in self._external_id_by_uri:
+            external_id = self._external_id_by_uri[uri]
         else:
-            space = self._space_by_instance_uri[uri]
-            if uri in self._external_id_by_uri:
-                external_id = self._external_id_by_uri[uri]
-            else:
-                external_id = remove_namespace_from_uri(uri)
+            external_id = remove_namespace_from_uri(uri)
 
         if external_id and self._unquote_external_ids:
             external_id = urllib.parse.unquote(external_id)
-        if space and external_id:
-            return InstanceId(space, external_id)
-        if error is None:
-            raise ValueError(f"Bug in neat. Failed to create instance ID and determine error for {uri!r}")
-        if stop_on_exception:
-            raise error
-        return error
+        return InstanceId(space, external_id)
 
     def _get_required_capabilities(self) -> list[Capability]:
         return [
@@ -696,7 +589,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     DataModelInstancesAcl.Action.Write_Properties,
                     DataModelInstancesAcl.Action.Read,
                 ],
-                scope=DataModelInstancesAcl.Scope.SpaceID([self._instance_space]),
+                scope=DataModelInstancesAcl.Scope.SpaceID(sorted(set(self._space_by_instance_uri.values()))),
             )
         ]