cognite-neat 0.87.0__tar.gz → 0.87.3__tar.gz

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.87.0
+Version: 0.87.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -21,7 +21,7 @@ Provides-Extra: oxi
 Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.52.0,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
 Requires-Dist: deepdiff
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
 Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"

cognite_neat-0.87.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.87.3"

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/app/api/configuration.py

@@ -9,8 +9,6 @@ from fastapi import FastAPI
 from cognite import neat
 from cognite.neat.config import Config, configure_logging, copy_examples_to_directory, create_data_dir_structure
 from cognite.neat.constants import PACKAGE_DIRECTORY
-from cognite.neat.utils.cdf import ServiceCogniteClient
-from cognite.neat.utils.utils import get_cognite_client_from_config, get_cognite_client_from_token
 from cognite.neat.workflows.cdf_store import CdfStore
 from cognite.neat.workflows.manager import WorkflowManager
 from cognite.neat.workflows.triggers import TriggerManager
@@ -36,14 +34,7 @@ class NeatApp:
             self.config = config
         logging.info("Initializing global objects")
         if not self.cdf_client:
-            if isinstance(self.config.cdf_client, ServiceCogniteClient):
-                if self.config.cdf_client.client_id:
-                    self.cdf_client = get_cognite_client_from_config(self.config.cdf_client)
-                else:
-                    # If no client_id is provided, we assume that the token is provided instead of secret.
-                    self.cdf_client = get_cognite_client_from_token(self.config.cdf_client)
-            else:
-                raise ValueError("Only ServiceCogniteClient is supported at the moment.")
+            self.cdf_client = self.config.cdf_auth_config.get_client()
         self.cdf_store = CdfStore(
             self.cdf_client,
             self.config.cdf_default_dataset_id,

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/app/api/routers/data_exploration.py

@@ -17,7 +17,7 @@ from cognite.neat.app.api.data_classes.rest import (
 from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
 from cognite.neat.app.api.utils.query_templates import query_templates
 from cognite.neat.legacy.graph.transformations import query_generator
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 
 router = APIRouter()

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/config.py

@@ -4,14 +4,14 @@ import os
 import shutil
 import sys
 from pathlib import Path
-from typing import Literal, cast
+from typing import Any, Literal, cast
 
 import yaml
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_serializer, model_validator
 from yaml import safe_load
 
 from cognite.neat.constants import EXAMPLE_GRAPHS, EXAMPLE_RULES, EXAMPLE_WORKFLOWS
-from cognite.neat.utils.cdf import InteractiveCogniteClient, ServiceCogniteClient
+from cognite.neat.utils.auth import EnvironmentVariables
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -41,7 +41,7 @@ class WorkflowsStoreType(StrEnum):
     URL = "url"
 
 
-class Config(BaseModel):
+class Config(BaseModel, arbitrary_types_allowed=True):
     workflows_store_type: WorkflowsStoreType = WorkflowsStoreType.FILE
     data_store_path: Path = Field(default_factory=lambda: Path.cwd() / "data")
 
@@ -51,7 +51,7 @@ class Config(BaseModel):
         default=None,
     )
 
-    cdf_client: InteractiveCogniteClient | ServiceCogniteClient = ServiceCogniteClient()
+    cdf_auth_config: EnvironmentVariables = Field(default_factory=EnvironmentVariables.default)
     cdf_default_dataset_id: int = 0
     load_examples: bool = True
 
@@ -60,9 +60,64 @@ class Config(BaseModel):
     download_workflows_from_cdf: bool = Field(
         default=False, description="Downloads all workflows from CDF automatically and stores them locally"
     )
-
     stop_on_error: bool = False
 
+    @model_validator(mode="before")
+    def backwards_compatible(cls, data: Any):
+        if not isinstance(data, dict):
+            return data
+        if "cdf_client" in data:
+            cdf_client = data["cdf_client"]
+            if isinstance(cdf_client, dict):
+                if "base_url" in cdf_client:
+                    base_url = cdf_client["base_url"]
+                    cluster = base_url.removeprefix("https://").removesuffix(".cognitedata.com")
+                else:
+                    base_url, cluster = "Missing", "Missing"
+                if "scopes" in cdf_client:
+                    scopes = cdf_client["scopes"]
+                    if isinstance(scopes, list):
+                        scopes = ",".join(scopes)
+                else:
+                    scopes = "Missing"
+                data["cdf_auth_config"] = EnvironmentVariables(
+                    CDF_PROJECT=cdf_client.get("project", "Missing"),
+                    CDF_CLUSTER=cluster,
+                    CDF_URL=base_url,
+                    IDP_CLIENT_ID=cdf_client.get("client_id", "Missing"),
+                    IDP_CLIENT_SECRET=cdf_client.get("client_secret", "Missing"),
+                    IDP_TOKEN_URL=cdf_client.get("token_url", "Missing"),
+                    IDP_SCOPES=scopes,
+                    CDF_TIMEOUT=int(cdf_client.get("timeout", 60)),
+                    CDF_MAX_WORKERS=int(cdf_client.get("max_workers", 3)),
+                )
+        return data
+
+    @staticmethod
+    @field_serializer("cdf_auth_config", when_used="always", return_type=dict)
+    def backwards_compatible_serialize(cdf_auth_config: EnvironmentVariables) -> dict[str, Any]:
+        output: dict[str, Any] = {}
+        config = cdf_auth_config
+        if config.CDF_PROJECT not in {"Missing", "NOT SET"}:
+            output["project"] = config.CDF_PROJECT
+        if config.CDF_CLUSTER not in {"Missing", "NOT SET"}:
+            output["cluster"] = config.CDF_CLUSTER
+        if config.CDF_URL:
+            output["base_url"] = config.CDF_URL
+        if config.IDP_CLIENT_ID:
+            output["client_id"] = config.IDP_CLIENT_ID
+        if config.IDP_CLIENT_SECRET:
+            output["client_secret"] = config.IDP_CLIENT_SECRET
+        if config.IDP_TOKEN_URL:
+            output["token_url"] = config.IDP_TOKEN_URL
+        if config.IDP_SCOPES:
+            output["scopes"] = config.idp_scopes
+        if config.CDF_TIMEOUT:
+            output["timeout"] = config.CDF_TIMEOUT
+        if config.CDF_MAX_WORKERS:
+            output["max_workers"] = config.CDF_MAX_WORKERS
+        return output
+
     @property
     def _dir_suffix(self) -> str:
         is_test_running = "pytest" in sys.modules
@@ -93,7 +148,7 @@ class Config(BaseModel):
 
     def to_yaml(self, filepath: Path):
         # Parse as json to avoid Path and Enum objects
-        dump = json.loads(self.json())
+        dump = json.loads(self.model_dump_json())
 
         with filepath.open("w") as f:
             yaml.safe_dump(dump, f)
@@ -101,15 +156,27 @@ class Config(BaseModel):
     @classmethod
     def from_env(cls) -> Self:
         missing = "Missing"
-        cdf_config = ServiceCogniteClient(
-            project=os.environ.get("NEAT_CDF_PROJECT", missing),
-            client_id=os.environ.get("NEAT_CDF_CLIENT_ID", missing),
-            client_secret=os.environ.get("NEAT_CDF_CLIENT_SECRET", missing),
-            base_url=os.environ.get("NEAT_CDF_BASE_URL", missing),
-            token_url=os.environ.get("NEAT_CDF_TOKEN_URL", missing),
-            scopes=[os.environ.get("NEAT_CDF_SCOPES", missing)],
-            timeout=int(os.environ.get("NEAT_CDF_CLIENT_TIMEOUT", "60")),
-            max_workers=int(os.environ.get("NEAT_CDF_CLIENT_MAX_WORKERS", "3")),
+        # This is to be backwards compatible with the old config
+
+        base_url: str | None = None
+        if "NEAT_CDF_BASE_URL" in os.environ:
+            base_url = os.environ["NEAT_CDF_BASE_URL"]
+        if isinstance(base_url, str):
+            cluster = base_url.removeprefix("https://").removesuffix(".cognitedata.com")
+        else:
+            cluster = missing
+        variables = EnvironmentVariables(
+            CDF_PROJECT=os.environ.get("NEAT_CDF_PROJECT", missing),
+            CDF_CLUSTER=cluster,
+            CDF_URL=base_url,
+            IDP_CLIENT_ID=os.environ.get("NEAT_CDF_CLIENT_ID"),
+            IDP_CLIENT_SECRET=os.environ.get("NEAT_CDF_CLIENT_SECRET"),
+            IDP_TOKEN_URL=os.environ.get("NEAT_CDF_TOKEN_URL"),
+            IDP_SCOPES=os.environ.get("NEAT_CDF_SCOPES"),
+            CDF_TIMEOUT=int(os.environ["NEAT_CDF_CLIENT_TIMEOUT"] if "NEAT_CDF_CLIENT_TIMEOUT" in os.environ else 60),
+            CDF_MAX_WORKERS=int(
+                os.environ["NEAT_CDF_CLIENT_MAX_WORKERS"] if "NEAT_CDF_CLIENT_MAX_WORKERS" in os.environ else 3
+            ),
         )
 
         if workflow_downloader_filter_value := os.environ.get("NEAT_WORKFLOW_DOWNLOADER_FILTER", None):
@@ -118,7 +185,7 @@ class Config(BaseModel):
             workflow_downloader_filter = None
 
         return cls(
-            cdf_client=cdf_config,
+            cdf_auth_config=variables,
             workflows_store_type=os.environ.get(  # type: ignore[arg-type]
                 "NEAT_WORKFLOWS_STORE_TYPE", WorkflowsStoreType.FILE
             ),

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
+from cognite.neat.utils.auxiliary import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class EventsExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -13,7 +13,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class FilesExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -10,7 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -11,7 +11,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class RelationshipsExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class SequencesExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class TimeSeriesExtractor(BaseExtractor):

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_dexpi.py

@@ -9,7 +9,7 @@ from rdflib import RDF, RDFS, XSD, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import as_neat_compliant_uri
+from cognite.neat.utils.rdf_ import as_neat_compliant_uri
 from cognite.neat.utils.xml_ import get_children, iterate_tree
 
 DEXPI = Namespace("http://sandbox.dexpi.org/rdl/")

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -17,7 +17,7 @@ from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
 

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/loaders/_rdf2asset.py

@@ -1,11 +1,14 @@
-from collections import defaultdict
+import json
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, fields
 from pathlib import Path
+from typing import cast
 
+import yaml
 from cognite.client import CogniteClient
 from cognite.client.data_classes import AssetWrite
-from cognite.client.data_classes.capabilities import Capability
+from cognite.client.data_classes.capabilities import AssetsAcl, Capability
+from cognite.client.exceptions import CogniteAPIError
 
 from cognite.neat.graph._tracking.base import Tracker
 from cognite.neat.graph._tracking.log import LogTracker
@@ -81,10 +84,20 @@ class AssetLoader(CDFLoader[AssetWrite]):
         self.rules = rules
         self.data_set_id = data_set_id
         self.use_labels = use_labels
-        self.use_orphanage = use_orphanage
 
-        self.orphanage_external_id = (
-            f"{asset_external_id_prefix or ''}orphanage-{data_set_id}" if use_orphanage else None
+        self.orphanage = (
+            AssetWrite.load(
+                {
+                    "dataSetId": self.data_set_id,
+                    "externalId": (
+                        f"{asset_external_id_prefix or ''}orphanage-{data_set_id}" if use_orphanage else None
+                    ),
+                    "name": "Orphanage",
+                    "description": "Orphanage for assets whose parents do not exist",
+                }
+            )
+            if use_orphanage
+            else None
         )
 
         self.asset_external_id_prefix = asset_external_id_prefix
@@ -93,23 +106,6 @@ class AssetLoader(CDFLoader[AssetWrite]):
         self._issues = NeatIssueList[NeatIssue](create_issues or [])
         self._tracker: type[Tracker] = tracker or LogTracker
 
-    def _create_validation_classes(self) -> None:
-        # need to get back class-property pairs where are definition of
-        # asset implementations, extend InformationRulesAnalysis make it generic
-
-        # by default if there is not explicitly stated external_id
-        # use rdf:type and drop the prefix
-
-        # based on those create pydantic model AssetDefinition
-        # which will have .to_asset_write()
-
-        raise NotImplementedError("Not implemented yet, this is placeholder")
-
-    def categorize_assets(self, client: CogniteClient) -> None:
-        """Categorize assets to those to be created, updated, decommissioned, or resurrected"""
-
-        raise NotImplementedError("Not implemented yet, this is placeholder")
-
     def _load(self, stop_on_exception: bool = False) -> Iterable[AssetWrite | NeatIssue | type[_END_OF_CLASS]]:
         if self._issues.has_errors and stop_on_exception:
             raise self._issues.as_exception()
@@ -120,50 +116,79 @@ class AssetLoader(CDFLoader[AssetWrite]):
             # There should already be an error in this case.
             return
 
-        class_ids = [repr(class_.class_.id) for class_ in self.rules.classes]
-        tracker = self._tracker(type(self).__name__, class_ids, "classes")
+        ordered_classes = AssetAnalysis(self.rules).class_topological_sort()
 
-        for class_ in self.rules.classes:
-            tracker.start(repr(class_.class_.id))
+        tracker = self._tracker(
+            type(self).__name__,
+            [repr(class_.id) for class_ in ordered_classes],
+            "classes",
+        )
+
+        processed_instances = set()
+
+        if self.orphanage:
+            yield self.orphanage
+            processed_instances.add(self.orphanage.external_id)
 
-            property_renaming_config = AssetAnalysis(self.rules).define_property_renaming_config(class_.class_)
+        for class_ in ordered_classes:
+            tracker.start(repr(class_.id))
 
-            for identifier, properties in self.graph_store.read(class_.class_.suffix):
+            property_renaming_config = AssetAnalysis(self.rules).define_property_renaming_config(class_)
+
+            for identifier, properties in self.graph_store.read(class_.suffix):
                 fields = _process_properties(properties, property_renaming_config)
                 # set data set id and external id
-                fields["data_set_id"] = self.data_set_id
-                fields["external_id"] = identifier
+                fields["dataSetId"] = self.data_set_id
+                fields["externalId"] = identifier
+
+                # check on parent
+                if "parentExternalId" in fields and fields["parentExternalId"] not in processed_instances:
+                    error = loader_issues.InvalidInstanceError(
+                        type_="asset",
+                        identifier=identifier,
+                        reason=(
+                            f"Parent asset {fields['parentExternalId']} does not exist or failed creation"
+                            f""" {
+                                f', moving the asset {identifier} under orphanage {self.orphanage.external_id}'
+                                if self.orphanage
+                                else ''}"""
+                        ),
+                    )
+                    tracker.issue(error)
+                    if stop_on_exception:
+                        raise error.as_exception()
+                    yield error
+
+                    # if orphanage is set asset will use orphanage as parent
+                    if self.orphanage:
+                        fields["parentExternalId"] = self.orphanage.external_id
+
+                    # otherwise asset will be skipped
+                    else:
+                        continue
 
                 try:
                     yield AssetWrite.load(fields)
+                    processed_instances.add(identifier)
                 except KeyError as e:
                     error = loader_issues.InvalidInstanceError(type_="asset", identifier=identifier, reason=str(e))
                     tracker.issue(error)
                     if stop_on_exception:
                         raise error.as_exception() from e
                     yield error
-            yield _END_OF_CLASS
 
-    def load_to_cdf(self, client: CogniteClient, dry_run: bool = False) -> Sequence[AssetWrite]:
-        # generate assets
-        # check for circular asset hierarchy
-        # check for orphaned assets
-        # batch upsert of assets to CDF (otherwise we will hit the API rate limit)
-
-        raise NotImplementedError("Not implemented yet, this is placeholder")
-
-    @classmethod
-    def _check_for_circular_asset_hierarchy(cls, assets: list[AssetWrite]) -> None:
-        """Check for circular references in the asset rules"""
-        raise NotImplementedError("Not implemented yet, this is placeholder")
-
-    @classmethod
-    def _check_for_orphaned_assets(cls, assets: list[AssetWrite]) -> None:
-        """Check for circular references in the asset rules"""
-        raise NotImplementedError("Not implemented yet, this is placeholder")
+            yield _END_OF_CLASS
 
     def _get_required_capabilities(self) -> list[Capability]:
-        raise NotImplementedError("Not implemented yet, this is placeholder")
+        return [
+            AssetsAcl(
+                actions=[
+                    AssetsAcl.Action.Write,
+                    AssetsAcl.Action.Read,
+                ],
+                scope=AssetsAcl.Scope.DataSet([self.data_set_id]),
+            )
+        ]
 
     def _upload_to_cdf(
         self,
@@ -172,14 +197,45 @@ class AssetLoader(CDFLoader[AssetWrite]):
         dry_run: bool,
         read_issues: NeatIssueList,
     ) -> Iterable[UploadResult]:
-        raise NotImplementedError("Not implemented yet, this is placeholder")
+        try:
+            upserted = client.assets.upsert(items, mode="replace")
+        except CogniteAPIError as e:
+            result = UploadResult[str](name="Asset", issues=read_issues)
+            result.error_messages.append(str(e))
+            result.failed_upserted.update(item.as_id() for item in e.failed + e.unknown)
+            result.upserted.update(item.as_id() for item in e.successful)
+            yield result
+        else:
+            for asset in upserted:
+                result = UploadResult[str](name="asset", issues=read_issues)
+                result.upserted.add(cast(str, asset.external_id))
+                yield result
 
     def write_to_file(self, filepath: Path) -> None:
-        raise NotImplementedError("Not implemented yet, this is placeholder")
+        if filepath.suffix not in [".json", ".yaml", ".yml"]:
+            raise ValueError(f"File format {filepath.suffix} is not supported")
+        dumped: dict[str, list] = {"assets": []}
+        for item in self.load(stop_on_exception=False):
+            key = {
+                AssetWrite: "assets",
+                NeatIssue: "issues",
+                _END_OF_CLASS: "end_of_class",
+            }.get(type(item))
+            if key is None:
+                # This should never happen, and is a bug in neat
+                raise ValueError(f"Item {item} is not supported. This is a bug in neat please report it.")
+            if key == "end_of_class":
+                continue
+            dumped[key].append(item.dump())
+        with filepath.open("w", encoding=self._encoding, newline=self._new_line) as f:
+            if filepath.suffix == ".json":
+                json.dump(dumped, f, indent=2)
+            else:
+                yaml.safe_dump(dumped, f, sort_keys=False)
 
 
 def _process_properties(properties: dict[str, list[str]], property_renaming_config: dict[str, str]) -> dict:
-    metadata: dict[str, str] = defaultdict(str)
+    metadata: dict[str, str] = {}
     fields: dict[str, str | dict] = {}
 
     for original_property, values in properties.items():

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/loaders/_rdf2dms.py

@@ -8,11 +8,11 @@ from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
 from cognite.client.data_classes.capabilities import Capability, DataModelInstancesAcl
 from cognite.client.data_classes.data_modeling import ViewId
+from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
 from cognite.client.data_classes.data_modeling.ids import InstanceId
 from cognite.client.data_classes.data_modeling.views import SingleEdgeConnection
 from cognite.client.exceptions import CogniteAPIError
-from pydantic import ValidationInfo, create_model, field_validator
-from pydantic.main import Model
+from pydantic import BaseModel, ValidationInfo, create_model, field_validator
 
 from cognite.neat.graph._tracking import LogTracker, Tracker
 from cognite.neat.graph.issues import loader as loader_issues
@@ -20,8 +20,8 @@ from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
+from cognite.neat.utils.auxiliary import create_sha256_hash
 from cognite.neat.utils.upload import UploadResult
-from cognite.neat.utils.utils import create_sha256_hash
 
 from ._base import CDFLoader
 
@@ -140,7 +140,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
 
     def _create_validation_classes(
         self, view: dm.View
-    ) -> tuple[type[Model], dict[str, dm.EdgeConnection], NeatIssueList]:
+    ) -> tuple[type[BaseModel], dict[str, dm.EdgeConnection], NeatIssueList]:
         issues = NeatIssueList[NeatIssue]()
         field_definitions: dict[str, tuple[type, Any]] = {}
         edge_by_property: dict[str, dm.EdgeConnection] = {}
@@ -168,7 +168,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     if data_type == Json:
                         json_fields.append(prop_name)
                     python_type = data_type.python
-                if prop.type.is_list:
+                if isinstance(prop.type, ListablePropertyType) and prop.type.is_list:
                     python_type = list[python_type]
                 default_value: Any = prop.default_value
                 if prop.nullable:
@@ -223,7 +223,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         self,
         identifier: str,
         properties: dict[str, list[str]],
-        pydantic_cls: type[Model],
+        pydantic_cls: type[BaseModel],
         view_id: dm.ViewId,
     ) -> dm.InstanceApply:
         created = pydantic_cls.model_validate(properties)

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/queries/_base.py

@@ -8,7 +8,7 @@ from rdflib.query import ResultRow
 
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationRules
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._construct import build_construct_query
 
@@ -112,25 +112,34 @@ class Queries:
         property_values: dict[str, list[str]] = defaultdict(list)
 
         for subject, predicate, object_ in cast(list[ResultRow], self.graph.query(f"DESCRIBE <{instance_id}>")):
-            # We cannot include the RDF.type in case there is a neat:type property
-            # or if the object is empty
-            if predicate != RDF.type and object_.lower() not in [
+            if object_.lower() not in [
                 "",
                 "none",
                 "nan",
                 "null",
             ]:
                 # we are skipping deep validation with Pydantic to remove namespace here
-                # as it reduce time to process triples by 10-15x
-                identifier, property_, value = cast(  # type: ignore[misc]
-                    (str, str, str),
-                    remove_namespace_from_uri(*(subject, predicate, object_), validation="prefix"),
+                # as it reduces time to process triples by 10-15x
+                identifier, value = cast(  # type: ignore[misc]
+                    (str, str),
+                    remove_namespace_from_uri(*(subject, object_), validation="prefix"),
                 )  # type: ignore[misc, index]
 
-                if property_renaming_config:
-                    property_ = property_renaming_config.get(property_, property_)
+                # use-case: calling describe without renaming properties
+                # losing the namespace from the predicate!
+                if not property_renaming_config and predicate != RDF.type:
+                    property_values[remove_namespace_from_uri(predicate, validation="prefix")].append(value)
+
+                # use-case: calling describe with renaming properties
+                # renaming the property to the new name, if the property is defined
+                # in the RULES sheet
+                elif property_renaming_config and (property_ := property_renaming_config.get(predicate, None)):
+                    property_values[property_].append(value)
+
+                # use-case: skip the property if it is not defined in property_renaming_config
+                else:
+                    continue
 
-                property_values[property_].append(value)
         if property_values:
             return (
                 identifier,

{cognite_neat-0.87.0 → cognite_neat-0.87.3}/cognite/neat/graph/queries/_construct.py

@@ -5,15 +5,15 @@ from rdflib import Graph, URIRef
 
 from cognite.neat.rules.analysis import InformationAnalysis
 from cognite.neat.rules.models._rdfpath import (
-    AllReferences,
     Hop,
     RDFPath,
+    SelfReferenceProperty,
     SingleProperty,
     Traversal,
 )
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationProperty, InformationRules
-from cognite.neat.utils.utils import most_occurring_element
+from cognite.neat.utils.collection_ import most_occurring_element
 
 from ._shared import Triple, hop2property_path
 
@@ -121,7 +121,7 @@ def to_construct_triples(
         templates.append(graph_template_triple)
 
         # use case AllReferences: binding instance to certain rdf property
-        if isinstance(traversal, AllReferences):
+        if isinstance(traversal, SelfReferenceProperty):
             graph_pattern_triple = Triple(
                 subject="BIND(?instance",
                 predicate="AS",