cognite-neat 0.101.0__py3-none-any.whl → 0.103.0__py3-none-any.whl

cognite/neat/_rules/exporters/_rules2excel.py

@@ -34,8 +34,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
     Args:
         styling: The styling to use for the Excel file. Defaults to "default". See below for details
             on the different styles.
-        output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
-            this role formate before being written to excel. If not provided, the role from the rules will be used.
         new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
             rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
             new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
@@ -119,7 +117,11 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
 
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
-            sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
+
+            if headers[0] == "Neat ID":
+                # Move the Neat ID to the end of the columns
+                headers = headers[1:] + ["Neat ID"]
+
             sheet.append(headers)
 
             fill_colors = itertools.cycle(["CADCFC", "FFFFFF"])
@@ -127,6 +129,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
             last_class: str | None = None
             item: dict[str, Any]
             for item in dumped_rules.get(sheet_name) or []:
+                if "Neat ID" in item:
+                    # Move the Neat ID to the end of the columns
+                    item["Neat ID"] = item.pop("Neat ID")
                 row = list(item.values())
                 class_ = row[0]
 
@@ -152,12 +157,13 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 # This freezes all rows above the given row
                 sheet.freeze_panes = sheet["A3"]
 
-                sheet["A1"].alignment = Alignment(horizontal="center")
+                sheet["A1"].alignment = Alignment(horizontal="left")
 
             if self._styling_level > 1:
                 # Make the header row bold, larger, and colored
-                sheet["A1"].font = Font(bold=True, size=20)
-                sheet["A1"].fill = PatternFill(fgColor="FFC000", patternType="solid")
+                for cell, *_ in sheet.iter_cols(min_row=1, max_row=1, min_col=1, max_col=len(headers)):
+                    cell.font = Font(bold=True, size=20)
+                    cell.fill = PatternFill(fgColor="FFC000", patternType="solid")
                 for cell in sheet["2"]:
                     cell.font = Font(bold=True, size=14)
 

cognite/neat/_rules/importers/__init__.py

@@ -7,14 +7,14 @@ from ._yaml2rules import YAMLImporter
 
 __all__ = [
     "BaseImporter",
-    "OWLImporter",
-    "IMFImporter",
     "DMSImporter",
+    "DTDLImporter",
     "ExcelImporter",
     "GoogleSheetImporter",
-    "DTDLImporter",
-    "YAMLImporter",
+    "IMFImporter",
     "InferenceImporter",
+    "OWLImporter",
+    "YAMLImporter",
 ]
 
 RulesImporters = (

cognite/neat/_rules/importers/_base.py

@@ -3,16 +3,18 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager, suppress
 from datetime import datetime
-from typing import Any, Generic, Literal
+from typing import TYPE_CHECKING, Any, Generic, Literal
 
 from pydantic import ValidationError
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._issues import IssueList, NeatError, NeatWarning
 from cognite.neat._rules._shared import ReadRules, T_InputRules
-from cognite.neat._store._provenance import Agent as ProvenanceAgent
 from cognite.neat._utils.auxiliary import class_html_doc
 
+if TYPE_CHECKING:
+    from cognite.neat._store._provenance import Agent as ProvenanceAgent
+
 
 class BaseImporter(ABC, Generic[T_InputRules]):
     """
@@ -48,8 +50,10 @@ class BaseImporter(ABC, Generic[T_InputRules]):
         return class_html_doc(cls)
 
     @property
-    def agent(self) -> ProvenanceAgent:
+    def agent(self) -> "ProvenanceAgent":
         """Provenance agent for the importer."""
+        from cognite.neat._store._provenance import Agent as ProvenanceAgent
+
         return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
 
 

cognite/neat/_rules/importers/_dms2rules.py

@@ -1,4 +1,4 @@
-from collections import Counter
+from collections import Counter, defaultdict
 from collections.abc import Collection, Iterable, Sequence
 from datetime import datetime, timezone
 from pathlib import Path
@@ -222,12 +222,18 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         schema: DMSSchema,
         metadata: DMSInputMetadata | None = None,
     ) -> DMSInputRules:
+        enum_by_container_property = self._create_enum_collections(schema.containers.values())
+        enum_collection_by_container_property = {
+            key: enum_list[0].collection for key, enum_list in enum_by_container_property.items() if enum_list
+        }
+
         properties: list[DMSInputProperty] = []
         for view_id, view in schema.views.items():
             view_entity = ViewEntity.from_id(view_id)
-            class_entity = view_entity.as_class()
             for prop_id, prop in (view.properties or {}).items():
-                dms_property = self._create_dms_property(prop_id, prop, view_entity, class_entity)
+                dms_property = self._create_dms_property(
+                    prop_id, prop, view_entity, enum_collection_by_container_property
+                )
                 if dms_property is not None:
                     properties.append(dms_property)
 
@@ -237,8 +243,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
 
         metadata = metadata or DMSInputMetadata.from_data_model(data_model)
 
-        enum = self._create_enum_collections(schema.containers.values())
-
         return DMSInputRules(
             metadata=metadata,
             properties=properties,
@@ -248,7 +252,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
                 for view_id, view in schema.views.items()
             ],
             nodes=[DMSInputNode.from_node_type(node_type) for node_type in schema.node_types.values()],
-            enum=enum,
+            enum=[enum for enum_list in enum_by_container_property.values() for enum in enum_list] or None,
         )
 
     @classmethod
@@ -267,7 +271,11 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         )
 
     def _create_dms_property(
-        self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
+        self,
+        prop_id: str,
+        prop: ViewPropertyApply,
+        view_entity: ViewEntity,
+        enum_collection_by_container_property: dict[tuple[dm.ContainerId, str], str],
     ) -> DMSInputProperty | None:
         if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
             self.issue_list.append(
@@ -300,7 +308,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             )
             return None
 
-        value_type = self._get_value_type(prop, view_entity, prop_id)
+        value_type = self._get_value_type(prop, view_entity, prop_id, enum_collection_by_container_property)
         if value_type is None:
             return None
 
@@ -347,7 +355,11 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             return None
 
     def _get_value_type(
-        self, prop: ViewPropertyApply, view_entity: ViewEntity, prop_id
+        self,
+        prop: ViewPropertyApply,
+        view_entity: ViewEntity,
+        prop_id: str,
+        enum_collection_by_container_property: dict[tuple[dm.ContainerId, str], str],
     ) -> DataType | ViewEntity | DMSUnknownEntity | None:
         if isinstance(
             prop,
@@ -367,7 +379,16 @@ class DMSImporter(BaseImporter[DMSInputRules]):
             elif isinstance(container_prop.type, PropertyTypeWithUnit) and container_prop.type.unit:
                 return DataType.load(f"{container_prop.type._type}(unit={container_prop.type.unit.external_id})")
             elif isinstance(container_prop.type, DMSEnum):
-                return Enum(collection=ClassEntity(suffix=prop_id), unknownValue=container_prop.type.unknown_value)
+                collection = enum_collection_by_container_property.get(
+                    (prop.container, prop.container_property_identifier)
+                )
+                if collection is None:
+                    # This should never happen
+                    raise ValueError(
+                        f"BUG in Neat: Enum for {prop.container}.{prop.container_property_identifier} not found."
+                    )
+
+                return Enum(collection=ClassEntity(suffix=collection), unknownValue=container_prop.type.unknown_value)
             else:
                 return DataType.load(container_prop.type._type)
         else:
@@ -477,15 +498,26 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         return candidates[0]
 
     @staticmethod
-    def _create_enum_collections(containers: Collection[dm.ContainerApply]) -> list[DMSInputEnum] | None:
-        enum_collections: list[DMSInputEnum] = []
+    def _create_enum_collections(
+        containers: Collection[dm.ContainerApply],
+    ) -> dict[tuple[dm.ContainerId, str], list[DMSInputEnum]]:
+        enum_by_container_property: dict[tuple[dm.ContainerId, str], list[DMSInputEnum]] = defaultdict(list)
+
+        is_external_id_unique = len({container.external_id for container in containers}) == len(containers)
+
         for container in containers:
+            container_id = container.as_id()
             for prop_id, prop in container.properties.items():
-                if isinstance(prop.type, DMSEnum):
-                    for identifier, value in prop.type.values.items():
-                        enum_collections.append(
-                            DMSInputEnum(
-                                collection=prop_id, value=identifier, name=value.name, description=value.description
-                            )
+                if not isinstance(prop.type, DMSEnum):
+                    continue
+                if is_external_id_unique:
+                    collection = f"{container.external_id}.{prop_id}"
+                else:
+                    collection = f"{container.space}:{container.external_id}.{prop_id}"
+                for identifier, value in prop.type.values.items():
+                    enum_by_container_property[(container_id, prop_id)].append(
+                        DMSInputEnum(
+                            collection=collection, value=identifier, name=value.name, description=value.description
                         )
-        return enum_collections
+                    )
+        return enum_by_container_property

cognite/neat/_rules/importers/_rdf/__init__.py

@@ -2,4 +2,4 @@ from ._imf2rules import IMFImporter
 from ._inference2rules import InferenceImporter
 from ._owl2rules import OWLImporter
 
-__all__ = ["IMFImporter", "OWLImporter", "InferenceImporter"]
+__all__ = ["IMFImporter", "InferenceImporter", "OWLImporter"]

cognite/neat/_rules/importers/_rdf/_base.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from cognite.client import data_modeling as dm
 from rdflib import Graph, Namespace, URIRef
 
-from cognite.neat._constants import get_default_prefixes
+from cognite.neat._constants import get_default_prefixes_and_namespaces
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import FileReadError
 from cognite.neat._issues.errors._general import NeatValueError
@@ -97,7 +97,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             issue_list.append(FileReadError(filepath, str(e)))
 
         # bind key namespaces
-        for prefix, namespace in get_default_prefixes().items():
+        for prefix, namespace in get_default_prefixes_and_namespaces().items():
             graph.bind(prefix, namespace)
 
         return cls(

cognite/neat/_rules/models/__init__.py

@@ -17,16 +17,16 @@ VERIFIED_RULES_BY_ROLE: dict[RoleTypes, type[InformationRules] | type[DMSRules]]
 
 
 __all__ = [
+    "INPUT_RULES_BY_ROLE",
     "DMSInputRules",
-    "InformationInputRules",
-    "InformationRules",
     "DMSRules",
-    "INPUT_RULES_BY_ROLE",
     "DMSSchema",
+    "DataModelType",
+    "ExtensionCategory",
+    "InformationInputRules",
+    "InformationRules",
     "RoleTypes",
     "SchemaCompleteness",
-    "ExtensionCategory",
-    "DataModelType",
     "SheetList",
     "SheetRow",
 ]

cognite/neat/_rules/models/_base_rules.py

@@ -149,7 +149,7 @@ class BaseMetadata(SchemaModel):
 
     role: ClassVar[RoleTypes] = Field(description="Role of the person creating the data model")
     aspect: ClassVar[DataModelAspect] = Field(description="Aspect of the data model")
-    space: SpaceType = Field(alias="prefix", description="The space where the data model is defined")
+    space: SpaceType = Field(description="The space where the data model is defined")
     external_id: DataModelExternalIdType = Field(
         alias="externalId", description="External identifier for the data model"
     )

cognite/neat/_rules/models/dms/__init__.py

@@ -13,20 +13,20 @@ from ._rules_input import (
 from ._validation import DMSValidation
 
 __all__ = [
-    "DMSRules",
-    "DMSSchema",
-    "DMSMetadata",
-    "DMSView",
-    "DMSProperty",
     "DMSContainer",
-    "DMSNode",
     "DMSEnum",
-    "DMSInputRules",
-    "DMSInputMetadata",
-    "DMSInputView",
-    "DMSInputProperty",
     "DMSInputContainer",
-    "DMSInputNode",
     "DMSInputEnum",
+    "DMSInputMetadata",
+    "DMSInputNode",
+    "DMSInputProperty",
+    "DMSInputRules",
+    "DMSInputView",
+    "DMSMetadata",
+    "DMSNode",
+    "DMSProperty",
+    "DMSRules",
+    "DMSSchema",
     "DMSValidation",
+    "DMSView",
 ]

cognite/neat/_rules/models/dms/_validation.py

@@ -1,5 +1,6 @@
 import warnings
 from collections import Counter, defaultdict
+from collections.abc import Mapping
 from functools import lru_cache
 from typing import ClassVar
 
@@ -115,7 +116,9 @@ class DMSValidation:
             **ref_container_by_id,
         }
         all_views_by_id: dict[dm.ViewId, dm.ViewApply | dm.View] = {**dict(dms_schema.views.items()), **ref_view_by_id}
-        properties_by_ids = self._as_properties_by_ids(dms_schema, ref_view_by_id)
+        properties_by_ids = self._as_properties_by_ids(dms_schema.views, ref_view_by_id)
+        ref_properties_by_ids = self._as_properties_by_ids(ref_view_by_id, {})
+        all_properties_by_ids = {**ref_properties_by_ids, **properties_by_ids}
         view_properties_by_id = self._as_view_properties_by_id(properties_by_ids)
         parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
 
@@ -132,7 +135,9 @@ class DMSValidation:
         # SDK classes validation
         issue_list.extend(self._containers_are_proper_size(dms_schema))
         issue_list.extend(
-            self._validate_reverse_connections(properties_by_ids, all_containers_by_id, parents_view_ids_by_child_id)
+            self._validate_reverse_connections(
+                properties_by_ids, all_containers_by_id, parents_view_ids_by_child_id, all_properties_by_ids
+            )
         )
         issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
         issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
@@ -140,22 +145,22 @@ class DMSValidation:
 
     @staticmethod
     def _as_properties_by_ids(
-        dms_schema: DMSSchema, ref_view_by_id: dict[dm.ViewId, dm.View]
+        view_by_id: Mapping[dm.ViewId, dm.ViewApply] | Mapping[dm.ViewId, dm.View],
+        ref_view_by_id: dict[dm.ViewId, dm.View],
     ) -> dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty]:
         # Priority DMS schema properties.
         # No need to do long lookups in ref_views as these already contain all ancestor properties.
         properties_by_id: dict[tuple[ViewId, str], ViewPropertyApply | ViewProperty] = {}
-        for view in dms_schema.views.values():
-            view_id = view.as_id()
+        for view_id, view in view_by_id.items():
             for prop_id, prop in (view.properties or {}).items():
                 properties_by_id[(view_id, prop_id)] = prop
             if view.implements:
                 to_check = view.implements.copy()
                 while to_check:
                     parent_id = to_check.pop()
-                    if parent_id in dms_schema.views:
-                        # Priority DMS Schema properties
-                        parent_view = dms_schema.views[parent_id]
+                    if parent_id in view_by_id:
+                        # Priority of the DMS schema properties
+                        parent_view = view_by_id[parent_id]
                         for prop_id, prop in (parent_view.properties or {}).items():
                             if (view_id, prop_id) not in properties_by_id:
                                 properties_by_id[(view_id, prop_id)] = prop
@@ -406,6 +411,7 @@ class DMSValidation:
         view_property_by_property_id: dict[tuple[dm.ViewId, str], ViewPropertyApply | ViewProperty],
         containers_by_id: dict[dm.ContainerId, dm.ContainerApply | dm.Container],
         parents_by_view: dict[dm.ViewId, set[dm.ViewId]],
+        all_view_property_by_property_id: dict[tuple[dm.ViewId, str], ViewPropertyApply | ViewProperty],
     ) -> IssueList:
         issue_list = IssueList()
         # do not check for reverse connections in Cognite models
@@ -416,7 +422,7 @@ class DMSValidation:
             if not isinstance(prop_, ReverseDirectRelationApply | ReverseDirectRelation):
                 continue
             target_id = prop_.through.source, prop_.through.property
-            if target_id not in view_property_by_property_id:
+            if target_id not in all_view_property_by_property_id:
                 issue_list.append(
                     ReversedConnectionNotFeasibleError(
                         view_id,
@@ -430,7 +436,7 @@ class DMSValidation:
                 # Todo: How to handle this case? Should not happen if you created the model with Neat
                 continue
 
-            target_property = view_property_by_property_id[(target_id[0], target_id[1])]
+            target_property = all_view_property_by_property_id[(target_id[0], target_id[1])]
             # Validate that the target is a direct relation pointing to the view_id
             is_direct_relation = False
             if isinstance(target_property, dm.MappedProperty) and isinstance(target_property.type, dm.DirectRelation):

cognite/neat/_rules/models/entities/__init__.py

@@ -25,38 +25,38 @@ from ._types import CdfResourceEntityList, ClassEntityList, ContainerEntityList,
 from ._wrapped import DMSFilter, HasDataFilter, NodeTypeFilter, RawFilter, WrappedEntity
 
 __all__ = [
-    "Entity",
-    "UnitEntity",
-    "DMSVersionedEntity",
-    "RelationshipEntity",
-    "EdgeEntity",
-    "ViewEntity",
-    "ClassEntity",
-    "ViewEntityList",
-    "UnknownEntity",
-    "Unknown",
-    "Undefined",
-    "T_Entity",
     "AssetEntity",
     "AssetFields",
-    "ReverseConnectionEntity",
-    "MultiValueTypeInfo",
-    "DMSNodeEntity",
-    "DMSEntity",
+    "CdfResourceEntityList",
+    "ClassEntity",
+    "ClassEntityList",
     "ContainerEntity",
+    "ContainerEntityList",
+    "DMSEntity",
+    "DMSFilter",
+    "DMSNodeEntity",
     "DMSUnknownEntity",
+    "DMSVersionedEntity",
     "DataModelEntity",
+    "EdgeEntity",
+    "Entity",
+    "HasDataFilter",
+    "MultiValueTypeInfo",
+    "NodeTypeFilter",
+    "RawFilter",
     "ReferenceEntity",
-    "ClassEntityList",
-    "CdfResourceEntityList",
-    "ContainerEntityList",
+    "RelationshipEntity",
+    "ReverseConnectionEntity",
+    "T_Entity",
     "URLEntity",
-    "load_value_type",
-    "load_dms_value_type",
-    "load_connection",
+    "Undefined",
+    "UnitEntity",
+    "Unknown",
+    "UnknownEntity",
+    "ViewEntity",
+    "ViewEntityList",
     "WrappedEntity",
-    "NodeTypeFilter",
-    "DMSFilter",
-    "HasDataFilter",
-    "RawFilter",
+    "load_connection",
+    "load_dms_value_type",
+    "load_value_type",
 ]

cognite/neat/_rules/models/information/__init__.py

@@ -8,13 +8,13 @@ from ._rules_input import (
 from ._validation import InformationValidation
 
 __all__ = [
-    "InformationRules",
-    "InformationMetadata",
     "InformationClass",
-    "InformationProperty",
-    "InformationInputRules",
-    "InformationInputMetadata",
     "InformationInputClass",
+    "InformationInputMetadata",
     "InformationInputProperty",
+    "InformationInputRules",
+    "InformationMetadata",
+    "InformationProperty",
+    "InformationRules",
     "InformationValidation",
 ]

cognite/neat/_rules/models/information/_rules.py

@@ -7,7 +7,7 @@ from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic_core.core_schema import SerializationInfo
 from rdflib import Namespace, URIRef
 
-from cognite.neat._constants import get_default_prefixes
+from cognite.neat._constants import get_default_prefixes_and_namespaces
 from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
@@ -255,7 +255,7 @@ class InformationRules(BaseRules):
     classes: SheetList[InformationClass] = Field(alias="Classes", description="List of classes")
     prefixes: dict[str, Namespace] = Field(
         alias="Prefixes",
-        default_factory=get_default_prefixes,
+        default_factory=get_default_prefixes_and_namespaces,
         description="the definition of the prefixes that are used in the semantic data model",
     )
 
@@ -264,7 +264,7 @@ class InformationRules(BaseRules):
         if isinstance(values, dict):
             return {key: Namespace(value) if isinstance(value, str) else value for key, value in values.items()}
         elif values is None:
-            values = get_default_prefixes()
+            values = get_default_prefixes_and_namespaces()
         return values
 
     def as_dms_rules(self) -> "DMSRules":