cognite-neat 0.118.0__tar.gz → 0.119.1__tar.gz

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.118.0
+Version: 0.119.1
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_issues/_contextmanagers.py

@@ -20,7 +20,9 @@ def catch_warnings() -> Iterator[IssueList]:
             yield issues
         finally:
             if warning_logger:
-                issues.extend([from_warning(warning) for warning in warning_logger])
+                issues.extend(
+                    [from_warning(warning) for warning in warning_logger if from_warning(warning) is not None]
+                )
 
 
 @contextmanager

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_issues/_factory.py

@@ -7,7 +7,6 @@ from cognite.neat._issues._base import NeatError, NeatWarning
 from cognite.neat._utils.spreadsheet import SpreadsheetRead
 
 from .errors import NeatValueError, SpreadsheetError
-from .warnings import NeatValueWarning
 
 
 def from_pydantic_errors(
@@ -23,13 +22,11 @@ def from_pydantic_errors(
     ]
 
 
-def from_warning(warning: WarningMessage) -> NeatWarning:
+def from_warning(warning: WarningMessage) -> NeatWarning | None:
     if isinstance(warning.message, NeatWarning):
         return warning.message
-    message = f"{warning.category.__name__}: {warning.message!s}"
-    if warning.source:
-        message += f" Source: {warning.source}"
-    return NeatValueWarning(message)
+
+    return None
 
 
 def _from_pydantic_error(error: ErrorDetails, read_info_by_sheet: dict[str, SpreadsheetRead]) -> NeatError:

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_rules/analysis/_base.py

@@ -154,8 +154,12 @@ class RulesAnalysis:
     def _include_ancestors(parents_by_class: dict[T_Hashable, set[T_Hashable]]) -> None:
         # Topological sort to ensure that classes include all ancestors
         for class_entity in list(TopologicalSorter(parents_by_class).static_order()):
+            if class_entity not in parents_by_class:
+                continue
             parents_by_class[class_entity] |= {
-                grand_parent for parent in parents_by_class[class_entity] for grand_parent in parents_by_class[parent]
+                grand_parent
+                for parent in parents_by_class[class_entity]
+                for grand_parent in parents_by_class.get(parent, set())
             }
 
     def properties_by_class(

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_rules/transformers/_converters.py

@@ -9,6 +9,7 @@ from functools import cached_property
 from graphlib import CycleError, TopologicalSorter
 from typing import Any, ClassVar, Literal, TypeVar, cast, overload
 
+from cognite.client import data_modeling as dm
 from cognite.client.data_classes import data_modeling as dms
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, View, ViewId
 from cognite.client.utils.useful_types import SequenceNotStr
@@ -28,8 +29,8 @@ from cognite.neat._constants import (
 )
 from cognite.neat._issues import IssueList
 from cognite.neat._issues._factory import from_pydantic_errors
-from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._issues.warnings import NeatValueWarning
+from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError
+from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
 from cognite.neat._issues.warnings._models import (
     SolutionModelBuildOnTopOfCDMWarning,
 )
@@ -72,7 +73,7 @@ from cognite.neat._rules.models.information import (
 )
 from cognite.neat._utils.rdf_ import get_inheritance_path
 from cognite.neat._utils.spreadsheet import SpreadsheetRead
-from cognite.neat._utils.text import NamingStandardization, title, to_camel_case, to_words
+from cognite.neat._utils.text import NamingStandardization, humanize_collection, title, to_camel_case, to_words
 
 from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
 from ._verification import VerifyDMSRules
@@ -532,13 +533,17 @@ class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
     """Converts InformationRules to DMSRules."""
 
     def __init__(
-        self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "warning"] = "error"
+        self,
+        ignore_undefined_value_types: bool = False,
+        reserved_properties: Literal["error", "warning"] = "error",
+        client: NeatClient | None = None,
     ):
         self.ignore_undefined_value_types = ignore_undefined_value_types
         self.reserved_properties = reserved_properties
+        self.client = client
 
     def transform(self, rules: InformationRules) -> DMSRules:
-        return _InformationRulesConverter(rules).as_dms_rules(
+        return _InformationRulesConverter(rules, self.client).as_dms_rules(
             self.ignore_undefined_value_types, self.reserved_properties
         )
 
@@ -1357,9 +1362,10 @@ class MergeInformationRules(VerifiedRulesTransformer[InformationRules, Informati
 class _InformationRulesConverter:
     _start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
 
-    def __init__(self, information: InformationRules):
+    def __init__(self, information: InformationRules, client: NeatClient | None = None):
         self.rules = information
         self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
+        self.client = client
 
     def as_dms_rules(
         self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "warning"] = "error"
@@ -1398,9 +1404,29 @@ class _InformationRulesConverter:
             and (prop.property_ == "endNode" or prop.property_ == "end_node")
             and isinstance(prop.value_type, ClassEntity)
         }
+        ancestors_by_view: dict[ViewEntity, set[ViewEntity]] = {}
+        parents_by_class = RulesAnalysis(self.rules).parents_by_class(
+            include_ancestors=True, include_different_space=True
+        )
+        for cls_, parents in parents_by_class.items():
+            view_type = cls_.as_view_entity(default_space, default_version)
+            parent_views = {parent.as_view_entity(default_space, default_version) for parent in parents}
+            if view_type in ancestors_by_view:
+                ancestors_by_view[view_type].update(parent_views)
+            else:
+                ancestors_by_view[view_type] = parent_views
+
+        cognite_properties, cognite_containers, cognite_views = self._get_cognite_components()
+        for view in cognite_views:
+            if view in ancestors_by_view:
+                ancestors_by_view[view].update(cognite_views[view])
+            else:
+                ancestors_by_view[view] = cognite_views[view]
 
         properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
-        referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
+        used_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
+        used_cognite_containers: dict[ContainerEntity, DMSContainer] = {}
+
         for prop in self.rules.properties:
             if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
                 continue
@@ -1413,17 +1439,34 @@ class _InformationRulesConverter:
                 warnings.warn(NeatValueWarning(f"{msg} Skipping..."), stacklevel=2)
                 continue
 
-            dms_property = self._as_dms_property(
-                prop,
-                default_space,
-                default_version,
-                edge_classes,
-                edge_value_types_by_class_property_pair,
-                end_node_by_edge,
-            )
+            if cognite_property := self._find_cognite_property(
+                prop.property_, parents_by_class[prop.class_], cognite_properties
+            ):
+                dms_property = self._customize_cognite_property(
+                    prop,
+                    cognite_property,
+                    prop.class_,
+                    default_space,
+                    default_version,
+                    ancestors_by_view,
+                )
+                if dms_property.container:
+                    if dms_property.container not in used_cognite_containers:
+                        used_cognite_containers[dms_property.container] = cognite_containers[dms_property.container]
+            else:
+                # Not matching any parent.
+                dms_property = self._as_dms_property(
+                    prop,
+                    default_space,
+                    default_version,
+                    edge_classes,
+                    edge_value_types_by_class_property_pair,
+                    end_node_by_edge,
+                )
+                if dms_property.container:
+                    used_containers[dms_property.container][prop.class_] += 1
+
             properties_by_class[prop.class_].append(dms_property)
-            if dms_property.container:
-                referenced_containers[dms_property.container][prop.class_] += 1
 
         views: list[DMSView] = []
 
@@ -1443,11 +1486,11 @@ class _InformationRulesConverter:
         existing_containers: set[ContainerEntity] = set()
 
         containers: list[DMSContainer] = []
-        for container_entity, class_entities in referenced_containers.items():
+        for container_entity, class_entities in used_containers.items():
             if container_entity in existing_containers:
                 continue
             constrains = self._create_container_constraint(
-                class_entities, default_space, class_by_entity, referenced_containers
+                class_entities, default_space, class_by_entity, used_containers
             )
             most_used_class_entity = class_entities.most_common(1)[0][0]
             class_ = class_by_entity[most_used_class_entity]
@@ -1468,6 +1511,9 @@ class _InformationRulesConverter:
             )
             containers.append(container)
 
+        if used_cognite_containers:
+            containers.extend(used_cognite_containers.values())
+
         dms_rules = DMSRules(
             metadata=dms_metadata,
             properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
@@ -1479,6 +1525,35 @@ class _InformationRulesConverter:
 
         return dms_rules
 
+    def _get_cognite_components(
+        self,
+    ) -> tuple[
+        dict[tuple[ClassEntity, str], DMSProperty],
+        dict[ContainerEntity, DMSContainer],
+        dict[ViewEntity, set[ViewEntity]],
+    ]:
+        cognite_concepts = self._get_cognite_concepts()
+        cognite_properties: dict[tuple[ClassEntity, str], DMSProperty] = {}
+        cognite_containers: dict[ContainerEntity, DMSContainer] = {}
+        cognite_views: dict[ViewEntity, set[ViewEntity]] = {}
+        if cognite_concepts:
+            if self.client is None:
+                raise CDFMissingClientError(
+                    f"Cannot convert {self.rules.metadata.as_data_model_id()}. Missing Cognite Client."
+                    f"This is required as the data model is referencing cognite concepts in the implements"
+                    f"{humanize_collection(cognite_concepts)}"
+                )
+            cognite_rules = self._get_cognite_dms_rules(cognite_concepts, self.client)
+
+            cognite_properties = {
+                (dms_prop.view.as_class(), dms_prop.view_property): dms_prop for dms_prop in cognite_rules.properties
+            }
+            cognite_containers = {container.container: container for container in cognite_rules.containers or []}
+            cognite_views = RulesAnalysis(dms=cognite_rules).implements_by_view(
+                include_ancestors=True, include_different_space=True
+            )
+        return cognite_properties, cognite_containers, cognite_views
+
     @staticmethod
     def _create_container_constraint(
         class_entities: Counter[ClassEntity],
@@ -1573,6 +1648,74 @@ class _InformationRulesConverter:
 
         return dms_property
 
+    @staticmethod
+    def _customize_cognite_property(
+        prop: InformationProperty,
+        cognite_prop: DMSProperty,
+        class_: ClassEntity,
+        default_space: str,
+        default_version: str,
+        ancestors_by_view: dict[ViewEntity, set[ViewEntity]],
+    ) -> DMSProperty:
+        """Customize the cognite property to match the information property.
+        This means updating the name and description of the cognite property with the information property.
+        In addition, the value type can be updated given that the value type is matches the cognite property value
+        type or in the case of a View Value type a derivative of the cognite property value type.
+
+        Args:
+            prop: Information property
+            cognite_prop: Cognite property
+            class_: Class entity
+            default_space: The default space
+            default_version: The default version
+            ancestors_by_view: Ancestors by view
+
+        Returns:
+            DMSProperty: The customized cognite property
+
+        """
+        value_type: DataType | ViewEntity | DMSUnknownEntity = cognite_prop.value_type
+        if isinstance(prop.value_type, DataType) and prop.value_type != value_type:
+            warnings.warn(
+                PropertyOverwritingWarning(prop.property_, "property", "value type", (str(prop.value_type),)),
+                stacklevel=2,
+            )
+        elif isinstance(prop.value_type, DataType):
+            # User set the same value type as core concept.
+            pass
+        elif isinstance(prop.value_type, ClassEntity) and isinstance(cognite_prop.value_type, ViewEntity):
+            view_type = prop.value_type.as_view_entity(default_space, default_version)
+            ancestors = ancestors_by_view.get(view_type, set())
+            if view_type == cognite_prop.value_type or cognite_prop.value_type in ancestors:
+                value_type = view_type
+            else:
+                warnings.warn(
+                    NeatValueWarning(
+                        f"Invalid Value Type. The view {view_type} must implement "
+                        f"{humanize_collection(ancestors, bind_word='or')} "
+                        f"to be used as the Value Type in the {prop.class_!s}.{prop.property_}. "
+                        f"Skipping..."
+                    ),
+                    stacklevel=2,
+                )
+        else:
+            warnings.warn(
+                NeatValueWarning(
+                    f"Invalid Value Type. The {prop.value_type} is not supported as {prop.class_} implements"
+                    f"a cognite concepts. Will skip this, and use the {cognite_prop.value_type} instead."
+                ),
+                stacklevel=2,
+            )
+
+        return cognite_prop.model_copy(
+            update={
+                "view": class_.as_view_entity(default_space, default_version),
+                "name": prop.name or cognite_prop.name,
+                "description": prop.description or cognite_prop.description,
+                "value_type": value_type,
+            }
+        )
+
     @staticmethod
     def _get_connection(
         prop: InformationProperty,
@@ -1713,6 +1856,48 @@ class _InformationRulesConverter:
 
         return data_types.String()
 
+    def _get_cognite_concepts(self) -> set[ClassEntity]:
+        return {cls_.class_ for cls_ in self.rules.classes if str(cls_.class_.prefix) in COGNITE_SPACES} | {
+            parent
+            for cls_ in self.rules.classes
+            for parent in cls_.implements or []
+            if str(parent.prefix) in COGNITE_SPACES
+        }
+
+    @staticmethod
+    def _get_cognite_dms_rules(concepts: set[ClassEntity], client: NeatClient) -> DMSRules:
+        view_ids = [dm.ViewId(str(cls_.prefix), cls_.suffix, cls_.version) for cls_ in concepts]
+        views = client.loaders.views.retrieve(view_ids, format="read", include_connected=True, include_ancestor=True)
+        spaces = Counter(view.space for view in views)
+        space = spaces.most_common(1)[0][0]
+        model: dm.DataModel[dm.View] = dm.DataModel(
+            space=space,
+            external_id="CognitePlaceholderModel",
+            version="v1",
+            is_global=False,
+            last_updated_time=1,
+            created_time=1,
+            name=None,
+            description="This model is constructed to hold all properties/views/containers that are referenced"
+            "by the data model being converted to DMS. This model is not meant to be used for any other"
+            "purpose.",
+            views=list(views),
+        )
+        unverified = DMSImporter.from_data_model(client, model).to_rules()
+        if unverified.rules is None:
+            raise NeatValueError("Failed to create CogniteConcepts")
+        return unverified.rules.as_verified_rules()
+
+    @staticmethod
+    def _find_cognite_property(
+        property_: str, parents: set[ClassEntity], cognite_properties: dict[tuple[ClassEntity, str], DMSProperty]
+    ) -> DMSProperty | None:
+        """Find the parent class that has the property in the cognite properties"""
+        for parent in parents:
+            if (parent, property_) in cognite_properties:
+                return cognite_properties[(parent, property_)]
+        return None
+
 
 class _DMSRulesConverter:
     def __init__(self, dms: DMSRules, instance_namespace: Namespace | None = None) -> None:

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_session/_base.py

@@ -168,7 +168,7 @@ class NeatSession:
         self._state._raise_exception_if_condition_not_met(
             "Convert to physical", has_dms_rules=False, has_information_rules=True
         )
-        converter = InformationToDMS(reserved_properties=reserved_properties)
+        converter = InformationToDMS(reserved_properties=reserved_properties, client=self._state.client)
 
         issues = self._state.rule_transform(converter)
 

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/cognite/neat/_utils/text.py

@@ -192,7 +192,29 @@ def replace_non_alphanumeric_with_underscore(text: str) -> str:
     return re.sub(r"\W+", "_", text)
 
 
-def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) -> str:
+def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bind_word: str = "and") -> str:
+    """Convert a collection of items to a human-readable string.
+
+    Args:
+        collection: The collection of items to convert.
+        sort: Whether to sort the collection before converting. Default is True.
+        bind_word: The word to use to bind the last two items. Default is "and".
+
+    Returns:
+        A human-readable string of the collection.
+
+    Examples:
+        >>> humanize_collection(["b", "c", "a"])
+        'a, b and c'
+        >>> humanize_collection(["b", "c", "a"], sort=False)
+        'b, c and a'
+        >>> humanize_collection(["a", "b"])
+        'a and b'
+        >>> humanize_collection(["a"])
+        'a'
+        >>> humanize_collection([])
+
+    """
     if not collection:
         return ""
     elif len(collection) == 1:
@@ -204,7 +226,7 @@ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) ->
     else:
         sequence = list(strings)
 
-    return f"{', '.join(sequence[:-1])} and {sequence[-1]}"
+    return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
 
 
 class NamingStandardization:

cognite_neat-0.119.1/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.119.1"
+__engine__ = "^2.0.4"

{cognite_neat-0.118.0 → cognite_neat-0.119.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-neat"
-version = "0.118.0"
+version = "0.119.1"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.118.0/cognite/neat/_version.py

@@ -1,2 +0,0 @@
-__version__ = "0.118.0"
-__engine__ = "^2.0.4"