cognite-neat 0.97.2__py3-none-any.whl → 0.98.0__py3-none-any.whl

cognite/neat/_rules/models/_base_rules.py

@@ -4,11 +4,23 @@ its sub-models and validators.
 
 from __future__ import annotations
 
+import math
 import sys
 import types
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Hashable, Iterator, MutableSequence, Sequence
-from typing import Annotated, Any, ClassVar, Literal, SupportsIndex, TypeVar, get_args, get_origin, overload
+from datetime import datetime
+from typing import (
+    Annotated,
+    Any,
+    ClassVar,
+    Literal,
+    SupportsIndex,
+    TypeVar,
+    get_args,
+    get_origin,
+    overload,
+)
 
 import pandas as pd
 from pydantic import (
@@ -23,15 +35,22 @@ from pydantic import (
 )
 from pydantic.main import IncEx
 from pydantic_core import core_schema
-
-from cognite.neat._rules.models._types import ClassEntityType, InformationPropertyType
+from rdflib import Namespace, URIRef
+
+from cognite.neat._constants import DEFAULT_NAMESPACE
+from cognite.neat._rules.models._types import (
+    ClassEntityType,
+    DataModelExternalIdType,
+    InformationPropertyType,
+    SpaceType,
+    StrListType,
+    VersionType,
+)
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
-    from typing import Self
 else:
     from backports.strenum import StrEnum
-    from typing_extensions import Self
 
 
 METADATA_VALUE_MAX_LENGTH = 5120
@@ -82,18 +101,18 @@ class DataModelType(StrEnum):
     enterprise = "enterprise"
 
 
+class DataModelAspect(StrEnum):
+    conceptual = "conceptual"
+    logical = "logical"
+    physical = "physical"
+
+
 class RoleTypes(StrEnum):
     domain_expert = "domain expert"
     information = "information architect"
-    asset = "asset architect"
     dms = "DMS Architect"
 
 
-class MatchType(StrEnum):
-    exact = "exact"
-    partial = "partial"
-
-
 class SchemaModel(BaseModel):
     model_config: ClassVar[ConfigDict] = ConfigDict(
         populate_by_name=True,
@@ -125,6 +144,46 @@ class BaseMetadata(SchemaModel):
     """
 
     role: ClassVar[RoleTypes]
+    aspect: ClassVar[DataModelAspect]
+    space: SpaceType = Field(alias="prefix")
+    external_id: DataModelExternalIdType = Field(alias="externalId")
+    version: VersionType
+
+    name: str | None = Field(
+        None,
+        description="Human readable name of the data model",
+        min_length=1,
+        max_length=255,
+    )
+
+    description: str | None = Field(None, min_length=1, max_length=1024)
+
+    creator: StrListType = Field(
+        description=(
+            "List of contributors to the data model creation, "
+            "typically information architects are considered as contributors."
+        ),
+    )
+
+    created: datetime = Field(
+        description=("Date of the data model creation"),
+    )
+
+    updated: datetime = Field(
+        description=("Date of the data model update"),
+    )
+
+    @field_validator("*", mode="before")
+    def strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        return value
+
+    @field_validator("description", mode="before")
+    def nan_as_none(cls, value):
+        if isinstance(value, float) and math.isnan(value):
+            return None
+        return value
 
     def to_pandas(self) -> pd.Series:
         """Converts Metadata to pandas Series."""
@@ -143,15 +202,30 @@ class BaseMetadata(SchemaModel):
     def include_role(self, serializer: Callable) -> dict:
         return {"role": self.role.value, **serializer(self)}
 
-    @abstractmethod
+    @property
+    def prefix(self) -> str:
+        return self.space
+
     def as_identifier(self) -> str:
-        """Returns a unique identifier for the metadata."""
-        raise NotImplementedError()
+        return f"{self.prefix}:{self.external_id}"
 
-    @abstractmethod
     def get_prefix(self) -> str:
-        """Returns the prefix for the metadata."""
-        raise NotImplementedError()
+        return self.prefix
+
+    @property
+    def identifier(self) -> URIRef:
+        """Globally unique identifier for the data model.
+
+        !!! note
+            Unlike namespace, the identifier does not end with "/" or "#".
+
+        """
+        return DEFAULT_NAMESPACE[f"data-model/verified/{self.aspect}/{self.space}/{self.external_id}/{self.version}"]
+
+    @property
+    def namespace(self) -> Namespace:
+        """Namespace for the data model used for the entities in the data model."""
+        return Namespace(f"{self.identifier}/")
 
 
 class BaseRules(SchemaModel, ABC):
@@ -169,7 +243,6 @@ class BaseRules(SchemaModel, ABC):
     """
 
     metadata: BaseMetadata
-    reference: Self | None = Field(None, alias="Reference")
 
     @classmethod
     def headers_by_sheet(cls, by_alias: bool = False) -> dict[str, list[str]]:
@@ -207,7 +280,6 @@ class BaseRules(SchemaModel, ABC):
     def dump(
         self,
         entities_exclude_defaults: bool = True,
-        as_reference: bool = False,
         mode: Literal["python", "json"] = "python",
         by_alias: bool = False,
         exclude: IncEx | None = None,
@@ -224,9 +296,6 @@ class BaseRules(SchemaModel, ABC):
                 For example, given a class that is dumped as 'my_prefix:MyClass', if the prefix for the rules
                 set in metadata.prefix = 'my_prefix', then this class will be dumped as 'MyClass' when this flag is set.
                 Defaults to True.
-            as_reference (bool, optional): Whether to dump as reference. For Information and DMS rules, this will
-                set the reference column/field to the reference of that entity. This is used in the ExcelExporter
-                to dump a reference model.
             mode: The mode in which `to_python` should run.
                 If mode is 'json', the output will only contain JSON serializable types.
                 If mode is 'python', the output may contain non-JSON-serializable Python objects.
@@ -242,25 +311,11 @@ class BaseRules(SchemaModel, ABC):
             if isinstance(value, SheetList):
                 value.sort(key=lambda x: x._identifier())
 
-        context: dict[str, Any] = {"as_reference": as_reference}
+        context: dict[str, Any] = {}
         if entities_exclude_defaults:
             context["metadata"] = self.metadata
 
-        exclude_input: IncEx | None
-        if self.reference is None:
-            exclude_input = exclude
-        else:
-            # If the rules has a reference, we dump that separately with the as_reference flag set to True.
-            # We don't want to include the reference in the main dump, so we exclude it here.
-            # This is to include whatever is in the exclude set from the user.
-            if isinstance(exclude, dict):
-                exclude_input = exclude.copy()
-                exclude_input["reference"] = {"__all__"}  # type: ignore[index]
-            elif isinstance(exclude, set):
-                exclude_input = exclude.copy()
-                exclude_input.add("reference")  # type: ignore[arg-type]
-            else:
-                exclude_input = {"reference"}
+        exclude_input: IncEx | None = exclude
 
         output = self.model_dump(
             mode=mode,
@@ -271,20 +326,7 @@ class BaseRules(SchemaModel, ABC):
             exclude_defaults=exclude_defaults,
             context=context,
         )
-        is_reference_user_excluded = isinstance(exclude, dict | set) and "reference" in exclude
-        if self.reference is not None and not is_reference_user_excluded:
-            # If the rules has a reference, we dump that separately with the as_reference flag set to True.
-            # Unless the user has explicitly excluded the reference.
-            output["Reference" if by_alias else "reference"] = self.reference.dump(
-                mode=mode,
-                by_alias=by_alias,
-                exclude=exclude,
-                exclude_none=exclude_none,
-                exclude_unset=exclude_unset,
-                exclude_defaults=exclude_defaults,
-                entities_exclude_defaults=entities_exclude_defaults,
-                as_reference=True,
-            )
+
         return output
 
 
@@ -293,6 +335,10 @@ class SheetRow(SchemaModel):
     def _identifier(self) -> tuple[Hashable, ...]:
         raise NotImplementedError()
 
+    def __repr__(self) -> str:
+        # Simplified representation of the object for debugging
+        return f"{self.__class__.__name__}({self._identifier()})"
+
 
 T_SheetRow = TypeVar("T_SheetRow", bound=SheetRow)
 

cognite/neat/_rules/models/dms/_exporter.py

@@ -1,7 +1,7 @@
 import warnings
 from collections import defaultdict
 from collections.abc import Collection, Hashable, Sequence
-from typing import Any, cast
+from typing import Any
 
 from cognite.client.data_classes import data_modeling as dm
 from cognite.client.data_classes.data_modeling.containers import BTreeIndex
@@ -18,10 +18,7 @@ from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundW
 from cognite.neat._issues.warnings.user_modeling import (
     EmptyContainerWarning,
     HasDataFilterOnNoPropertiesViewWarning,
-    HasDataFilterOnViewWithReferencesWarning,
-    NodeTypeFilterOnParentViewWarning,
 )
-from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat._rules.models.data_types import DataType, Double, Enum, Float
 from cognite.neat._rules.models.entities import (
     ClassEntity,
@@ -32,7 +29,6 @@ from cognite.neat._rules.models.entities import (
     EdgeEntity,
     HasDataFilter,
     NodeTypeFilter,
-    ReferenceEntity,
     ReverseConnectionEntity,
     UnitEntity,
     ViewEntity,
@@ -64,60 +60,19 @@ class _DMSExporter:
         self.include_pipeline = include_pipeline
         self.instance_space = instance_space
         self.rules = rules
-        self._ref_schema = rules.reference.as_schema() if rules.reference else None
+        self._ref_schema = None
         if self._ref_schema:
             # We skip version as that will always be missing in the reference
             self._ref_views_by_id = {
                 dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views.values()
             }
         else:
-            self._ref_views_by_id = {}
-
-        self.is_addition = (
-            rules.metadata.schema_ is SchemaCompleteness.extended
-            and rules.metadata.extension is ExtensionCategory.addition
-        )
-        self.is_reshape = (
-            rules.metadata.schema_ is SchemaCompleteness.extended
-            and rules.metadata.extension is ExtensionCategory.reshape
-        )
-        self.is_rebuild = (
-            rules.metadata.schema_ is SchemaCompleteness.extended
-            and rules.metadata.extension is ExtensionCategory.rebuild
-        )
+            self._ref_views_by_id = {}  # type: ignore
 
     def to_schema(self) -> DMSSchema:
         rules = self.rules
         container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.rules.properties))
 
-        # If we are reshaping or rebuilding, and there are no properties in the current rules, we will
-        # include those properties from the last rules.
-        if rules.last and (self.is_reshape or self.is_rebuild):
-            selected_views = {view.view for view in rules.views}
-            selected_properties = [
-                prop
-                for prop in rules.last.properties
-                if prop.view in selected_views and prop.view.as_id() not in view_properties_by_id
-            ]
-            self._update_with_properties(
-                selected_properties, container_properties_by_id, view_properties_by_id, include_new_containers=True
-            )
-
-        # We need to include the properties from the last rules as well to create complete containers and view
-        # depending on the type of extension.
-        if rules.last and self.is_addition:
-            selected_properties = [
-                prop for prop in rules.last.properties if (prop.view.as_id() in view_properties_by_id)
-            ]
-            self._update_with_properties(selected_properties, container_properties_by_id, view_properties_by_id)
-        elif rules.last and (self.is_reshape or self.is_rebuild):
-            selected_properties = [
-                prop
-                for prop in rules.last.properties
-                if prop.container and prop.container.as_id() in container_properties_by_id
-            ]
-            self._update_with_properties(selected_properties, container_properties_by_id, None)
-
         containers = self._create_containers(container_properties_by_id, rules.enum)  # type: ignore[arg-type]
 
         view_properties_with_ancestors_by_id = self._gather_properties_with_ancestors(
@@ -136,30 +91,11 @@ class _DMSExporter:
             node_types = NodeApplyDict([dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters])
 
         last_schema: DMSSchema | None = None
-        if self.rules.last:
-            last_schema = self.rules.last.as_schema()
-            # Remove the views that are in the current model, last + current should equal the full model
-            # without any duplicates
-            last_schema.views = ViewApplyDict(
-                {view_id: view for view_id, view in last_schema.views.items() if view_id not in views}
-            )
-            last_schema.containers = ContainerApplyDict(
-                {
-                    container_id: container
-                    for container_id, container in last_schema.containers.items()
-                    if container_id not in containers
-                }
-            )
 
         views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
-        if rules.last and self.is_addition:
-            views_not_in_model.update({view.view.as_id() for view in rules.last.views if not view.in_model})
-
         data_model = rules.metadata.as_data_model()
 
         data_model_views = [view_id for view_id in views if view_id not in views_not_in_model]
-        if last_schema and self.is_addition:
-            data_model_views.extend([view_id for view_id in last_schema.views if view_id not in views_not_in_model])
 
         # Sorting to ensure deterministic order
         data_model.views = sorted(data_model_views, key=lambda v: v.as_tuple())  # type: ignore[union-attr]
@@ -207,17 +143,8 @@ class _DMSExporter:
         view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
     ) -> tuple[ViewApplyDict, set[dm.NodeId]]:
         input_views = list(self.rules.views)
-        if self.rules.last:
-            existing = {view.view.as_id() for view in input_views}
-            modified_views = [
-                v
-                for v in self.rules.last.views
-                if v.view.as_id() in view_properties_by_id and v.view.as_id() not in existing
-            ]
-            input_views.extend(modified_views)
 
         views = ViewApplyDict([dms_view.as_view() for dms_view in input_views])
-        dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in input_views}
 
         for view_id, view in views.items():
             view.properties = {}
@@ -228,43 +155,9 @@ class _DMSExporter:
                 if view_property is not None:
                     view.properties[prop.view_property] = view_property
 
-        data_model_type = self.rules.metadata.data_model_type
         unique_node_types: set[dm.NodeId] = set()
-        parent_views = {parent for view in views.values() for parent in view.implements or []}
-        for view_id, view in views.items():
-            dms_view = dms_view_by_id.get(view_id)
-            dms_properties = view_properties_by_id.get(view_id, [])
-            view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
-
-            view.filter = view_filter.as_dms_filter()
-            if isinstance(view_filter, NodeTypeFilter):
-                unique_node_types.update(view_filter.nodes)
-                if view.as_id() in parent_views:
-                    warnings.warn(
-                        NodeTypeFilterOnParentViewWarning(view.as_id()),
-                        stacklevel=2,
-                    )
-
-            elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
-                if dms_view and isinstance(dms_view.reference, ReferenceEntity):
-                    references = {dms_view.reference.as_view_id()}
-                elif any(True for prop in dms_properties if isinstance(prop.reference, ReferenceEntity)):
-                    references = {
-                        prop.reference.as_view_id()
-                        for prop in dms_properties
-                        if isinstance(prop.reference, ReferenceEntity)
-                    }
-                else:
-                    continue
-                warnings.warn(
-                    HasDataFilterOnViewWithReferencesWarning(view.as_id(), frozenset(references)),
-                    stacklevel=2,
-                )
-
-            if data_model_type == DataModelType.enterprise:
-                # Enterprise Model needs to create node types for all views,
-                # as they are expected for the solution model.
-                unique_node_types.add(dm.NodeId(space=view.space, external_id=view.external_id))
+        for view in views.values():
+            unique_node_types.add(dm.NodeId(space=view.space, external_id=view.external_id))
 
         return views, unique_node_types
 
@@ -272,9 +165,6 @@ class _DMSExporter:
     def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
         if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
             return prop.connection.edge_type.as_reference()
-        elif isinstance(prop.reference, ReferenceEntity):
-            ref_view_prop = prop.reference.as_view_property_id()
-            return cls._create_edge_type_from_view_id(cast(dm.ViewId, ref_view_prop.source), ref_view_prop.property)
         elif isinstance(prop.value_type, ViewEntity):
             return cls._create_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
         else:
@@ -298,14 +188,6 @@ class _DMSExporter:
             enum_values_by_collection[enum_value.collection].append(enum_value)
 
         containers = list(self.rules.containers or [])
-        if self.rules.last:
-            existing = {container.container.as_id() for container in containers}
-            modified_containers = [
-                c
-                for c in self.rules.last.containers or []
-                if c.container.as_id() in container_properties_by_id and c.container.as_id() not in existing
-            ]
-            containers.extend(modified_containers)
 
         containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in containers])
         container_to_drop = set()
@@ -408,17 +290,6 @@ class _DMSExporter:
         views: Sequence[DMSView],
     ) -> dict[dm.ViewId, list[DMSProperty]]:
         all_view_properties_by_id = view_properties_by_id.copy()
-        if self.rules.reference:
-            # We need to include t
-            ref_view_properties_by_id = self._gather_properties(self.rules.reference.properties)[1]
-            for view_id, properties in ref_view_properties_by_id.items():
-                if view_id not in all_view_properties_by_id:
-                    all_view_properties_by_id[view_id] = properties
-                else:
-                    existing_properties = {prop._identifier() for prop in all_view_properties_by_id[view_id]}
-                    for prop in properties:
-                        if prop._identifier() not in existing_properties:
-                            all_view_properties_by_id[view_id].append(prop)
 
         view_properties_with_parents_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
         view_by_view_id = {view.view.as_id(): view for view in views}
@@ -482,37 +353,13 @@ class _DMSExporter:
         self,
         view: dm.ViewApply,
         dms_view: DMSView | None,
-        data_model_type: DataModelType,
-        dms_properties: list[DMSProperty],
-    ) -> DMSFilter:
+    ) -> DMSFilter | None:
         selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
 
         if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
             # Has Explicit Filter, use it
             return dms_view.filter_
 
-        if data_model_type == DataModelType.solution and selected_filter_name in [NodeTypeFilter.name, ""]:
-            if (
-                dms_view
-                and isinstance(dms_view.reference, ReferenceEntity)
-                and not dms_properties
-                and (ref_view := self._ref_views_by_id.get(dms_view.reference.as_view_id()))
-                and ref_view.filter
-            ):
-                # No new properties, only reference, reuse the reference filter
-                return DMSFilter.from_dms_filter(ref_view.filter)
-            else:
-                referenced_node_ids = {
-                    prop.reference.as_node_entity()
-                    for prop in dms_properties
-                    if isinstance(prop.reference, ReferenceEntity)
-                }
-                if dms_view and isinstance(dms_view.reference, ReferenceEntity):
-                    referenced_node_ids.add(dms_view.reference.as_node_entity())
-
-                if referenced_node_ids:
-                    return NodeTypeFilter(inner=list(referenced_node_ids))
-
         # Enterprise Model or (Solution + HasData)
         ref_containers = view.referenced_containers()
         if not ref_containers or selected_filter_name == HasDataFilter.name:
@@ -597,7 +444,7 @@ class _DMSExporter:
                 (
                     prop
                     for prop in view_properties_with_ancestors_by_id.get(source_view_id, [])
-                    if prop.property_ == reverse_prop_id
+                    if prop.view_property == reverse_prop_id
                 ),
                 None,
             )