cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

cognite/neat/_rules/models/information/_rules_input.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any, Literal
+from typing import Any
 
 import pandas as pd
 from rdflib import Namespace, URIRef
@@ -25,19 +25,16 @@ from ._rules import (
 
 @dataclass
 class InformationInputMetadata(InputComponent[InformationMetadata]):
-    schema_: Literal["complete", "partial", "extended"]
-    prefix: str
-    namespace: str
+    space: str
+    external_id: str
     version: str
     creator: str
-    data_model_type: Literal["solution", "enterprise"] = "enterprise"
-    extension: Literal["addition", "reshape", "rebuild"] = "addition"
     name: str | None = None
     description: str | None = None
     created: datetime | str | None = None
     updated: datetime | str | None = None
-    license: str | None = None
-    rights: str | None = None
+    physical: str | None = None
+    conceptual: str | None = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationMetadata]:
@@ -51,6 +48,25 @@ class InformationInputMetadata(InputComponent[InformationMetadata]):
             output["updated"] = datetime.now()
         return output
 
+    @property
+    def prefix(self) -> str:
+        return self.space
+
+    @property
+    def identifier(self) -> URIRef:
+        """Globally unique identifier for the data model.
+
+        !!! note
+            Unlike namespace, the identifier does not end with "/" or "#".
+
+        """
+        return DEFAULT_NAMESPACE[f"data-model/unverified/logical/{self.space}/{self.external_id}/{self.version}"]
+
+    @property
+    def namespace(self) -> Namespace:
+        """Namespace for the data model used for the entities in the data model."""
+        return Namespace(f"{self.identifier}/")
+
 
 @dataclass
 class InformationInputProperty(InputComponent[InformationProperty]):
@@ -59,12 +75,9 @@ class InformationInputProperty(InputComponent[InformationProperty]):
     value_type: DataType | ClassEntity | MultiValueTypeInfo | UnknownEntity | str
     name: str | None = None
     description: str | None = None
-    comment: str | None = None
     min_count: int | None = None
     max_count: int | float | None = None
     default: Any | None = None
-    reference: str | None = None
-    match_type: str | None = None
     transformation: str | None = None
     # Only used internally
     inherited: bool = False
@@ -85,10 +98,7 @@ class InformationInputClass(InputComponent[InformationClass]):
     class_: ClassEntity | str
     name: str | None = None
     description: str | None = None
-    comment: str | None = None
-    parent: str | list[ClassEntity] | None = None
-    reference: str | None = None
-    match_type: str | None = None
+    implements: str | list[ClassEntity] | None = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationClass]:
@@ -101,12 +111,12 @@ class InformationInputClass(InputComponent[InformationClass]):
     def dump(self, default_prefix: str, **kwargs) -> dict[str, Any]:  # type: ignore[override]
         output = super().dump()
         parent: list[ClassEntity] | None = None
-        if isinstance(self.parent, str):
-            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
-        elif isinstance(self.parent, list):
-            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.parent]
+        if isinstance(self.implements, str):
+            parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
+        elif isinstance(self.implements, list):
+            parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]
         output["Class"] = ClassEntity.load(self.class_, prefix=default_prefix)
-        output["Parent Class"] = parent
+        output["Implements"] = parent
         return output
 
 
@@ -116,8 +126,6 @@ class InformationInputRules(InputRules[InformationRules]):
     properties: list[InformationInputProperty] = field(default_factory=list)
     classes: list[InformationInputClass] = field(default_factory=list)
     prefixes: dict[str, Namespace] | None = None
-    last: "InformationInputRules | None" = None
-    reference: "InformationInputRules | None" = None
 
     @classmethod
     def _get_verified_cls(cls) -> type[InformationRules]:
@@ -125,26 +133,12 @@ class InformationInputRules(InputRules[InformationRules]):
 
     def dump(self) -> dict[str, Any]:
         default_prefix = self.metadata.prefix
-        reference: dict[str, Any] | None = None
-        if isinstance(self.reference, InformationInputRules):
-            reference = self.reference.dump()
-        elif isinstance(self.reference, InformationRules):
-            # We need to load through the InformationRulesInput to set the correct default space and version
-            reference = InformationInputRules.load(self.reference.model_dump()).dump()
-        last: dict[str, Any] | None = None
-        if isinstance(self.last, InformationInputRules):
-            last = self.last.dump()
-        elif isinstance(self.last, InformationRules):
-            # We need to load through the InformationRulesInput to set the correct default space and version
-            last = InformationInputRules.load(self.last.model_dump()).dump()
 
         return dict(
             Metadata=self.metadata.dump(),
             Properties=[prop.dump(default_prefix) for prop in self.properties],
             Classes=[class_.dump(default_prefix) for class_ in self.classes],
             Prefixes=self.prefixes,
-            Last=last,
-            Reference=reference,
         )
 
     def _repr_html_(self) -> str:
@@ -152,14 +146,11 @@ class InformationInputRules(InputRules[InformationRules]):
             "type": "Logical Data Model",
             "intended for": "Information Architect",
             "name": self.metadata.name,
-            "external_id": self.metadata.prefix,
+            "external_id": self.metadata.external_id,
+            "space": self.metadata.space,
             "version": self.metadata.version,
             "classes": len(self.classes),
             "properties": len(self.properties),
         }
 
         return pd.DataFrame([summary]).T.rename(columns={0: ""})._repr_html_()  # type: ignore
-
-    @property
-    def id_(self) -> URIRef:
-        return DEFAULT_NAMESPACE[f"data-model/unverified/info/{self.metadata.prefix}/{self.metadata.version}"]

cognite/neat/_rules/models/information/_validation.py

@@ -4,10 +4,9 @@ from typing import cast
 
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError, ResourceNotDefinedError
+from cognite.neat._issues.warnings._models import UndefinedClassWarning
 from cognite.neat._rules._constants import EntityTypes
-from cognite.neat._rules.models._base_rules import DataModelType, SchemaCompleteness
 from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
-from cognite.neat._utils.rdf_ import get_inheritance_path
 
 from ._rules import InformationRules
 
@@ -24,72 +23,60 @@ class InformationPostValidation:
         self.issue_list = IssueList()
 
     def validate(self) -> IssueList:
-        if self.metadata.schema_ == SchemaCompleteness.partial:
-            return self.issue_list
-
-        if self.metadata.data_model_type == DataModelType.solution and not self.rules.reference:
-            raise ValueError("Reference data model is missing")
-
-        if self.metadata.schema_ == SchemaCompleteness.extended and not self.rules.last:
-            raise ValueError("Last version is missing")
-
-        self._dangling_classes()
-        self._referenced_parent_classes_exist()
+        self._namespaces_reassigned()
+        self._classes_without_properties()
+        self._parent_class_defined()
         self._referenced_classes_exist()
         self._referenced_value_types_exist()
-        self._namespaces_reassigned()
 
         return self.issue_list
 
-    def _dangling_classes(self) -> None:
+    def _classes_without_properties(self) -> None:
         # needs to be complete for this validation to pass
         defined_classes = {class_.class_ for class_ in self.classes}
         referred_classes = {property_.class_ for property_ in self.properties}
         class_parent_pairs = self._class_parent_pairs()
-        dangling_classes = set()
 
         if classes_without_properties := defined_classes.difference(referred_classes):
             for class_ in classes_without_properties:
-                # USE CASE: class has no direct properties and no parents
-                if class_ not in class_parent_pairs:
-                    dangling_classes.add(class_)
                 # USE CASE: class has no direct properties and no parents with properties
-                elif class_ not in class_parent_pairs and not any(
-                    parent in referred_classes for parent in get_inheritance_path(class_, class_parent_pairs)
-                ):
-                    dangling_classes.add(class_)
-
-        for class_ in dangling_classes:
-            self.issue_list.append(
-                NeatValueError(f"Class {class_} has no properties and is not a parent of any class with properties")
-            )
+                # and it is a class in the prefix of data model, as long as it is in the
+                # same prefix, meaning same space
+                if not class_parent_pairs[class_] and class_.prefix == self.metadata.prefix:
+                    self.issue_list.append(
+                        ResourceNotDefinedError[ClassEntity](
+                            resource_type="class",
+                            identifier=class_,
+                            location="Classes sheet",
+                        )
+                    )
 
-    def _referenced_parent_classes_exist(self) -> None:
-        # needs to be complete for this validation to pass
+    def _parent_class_defined(self) -> None:
+        """This is a validation to check if the parent class of a class is defined in the classes sheet."""
         class_parent_pairs = self._class_parent_pairs()
         classes = set(class_parent_pairs.keys())
         parents = set(itertools.chain.from_iterable(class_parent_pairs.values()))
 
         if undefined_parents := parents.difference(classes):
             for parent in undefined_parents:
-                # Todo: include row and column number
-                self.issue_list.append(
-                    ResourceNotDefinedError[ClassEntity](
-                        resource_type="class",
-                        identifier=parent,
-                        location="Classes sheet",
+                if parent.prefix != self.metadata.prefix:
+                    self.issue_list.append(UndefinedClassWarning(class_id=str(parent)))
+                else:
+                    self.issue_list.append(
+                        ResourceNotDefinedError[ClassEntity](
+                            resource_type="class",
+                            identifier=parent,
+                            location="Classes sheet",
+                        )
                     )
-                )
 
     def _referenced_classes_exist(self) -> None:
         # needs to be complete for this validation to pass
         defined_classes = {class_.class_ for class_ in self.classes}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        classes_with_explicit_properties = {property_.class_ for property_ in self.properties}
 
         # USE CASE: models are complete
-        if self.metadata.schema_ == SchemaCompleteness.complete and (
-            missing_classes := referred_classes.difference(defined_classes)
-        ):
+        if missing_classes := classes_with_explicit_properties.difference(defined_classes):
             for class_ in missing_classes:
                 self.issue_list.append(
                     ResourceNotDefinedError[ClassEntity](
@@ -99,19 +86,6 @@ class InformationPostValidation:
                     )
                 )
 
-        # USE CASE: models are extended (user + last = complete)
-        if self.metadata.schema_ == SchemaCompleteness.extended:
-            defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
-            if missing_classes := referred_classes.difference(defined_classes):
-                for class_ in missing_classes:
-                    self.issue_list.append(
-                        ResourceNotDefinedError[ClassEntity](
-                            resource_type="class",
-                            identifier=class_,
-                            location="Classes sheet",
-                        )
-                    )
-
     def _referenced_value_types_exist(self) -> None:
         # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
         defined_classes = {class_.class_ for class_ in self.classes} | {UnknownEntity()}
@@ -121,10 +95,7 @@ class InformationPostValidation:
             if property_.type_ == EntityTypes.object_property
         }
 
-        # USE CASE: models are complete
-        if self.metadata.schema_ == SchemaCompleteness.complete and (
-            missing_value_types := referred_object_types.difference(defined_classes)
-        ):
+        if missing_value_types := referred_object_types.difference(defined_classes):
             # Todo: include row and column number
             for missing in missing_value_types:
                 self.issue_list.append(
@@ -135,56 +106,17 @@ class InformationPostValidation:
                     )
                 )
 
-        # USE CASE: models are extended (user + last = complete)
-        if self.metadata.schema_ == SchemaCompleteness.extended:
-            defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
-            if missing_value_types := referred_object_types.difference(defined_classes):
-                # Todo: include row and column number
-                for missing in missing_value_types:
-                    self.issue_list.append(
-                        ResourceNotDefinedError(
-                            resource_type="class",
-                            identifier=cast(ClassEntity, missing),
-                            location="Classes sheet",
-                        )
-                    )
-
     def _class_parent_pairs(self) -> dict[ClassEntity, list[ClassEntity]]:
-        class_subclass_pairs: dict[ClassEntity, list[ClassEntity]] = {}
-
+        class_parent_pairs: dict[ClassEntity, list[ClassEntity]] = {}
         classes = self.rules.model_copy(deep=True).classes
 
-        # USE CASE: Solution model being extended (user + last + reference = complete)
-        if (
-            self.metadata.schema_ == SchemaCompleteness.extended
-            and self.metadata.data_model_type == DataModelType.solution
-        ):
-            classes += (
-                cast(InformationRules, self.rules.last).model_copy(deep=True).classes
-                + cast(InformationRules, self.rules.reference).model_copy(deep=True).classes
-            )
-
-        # USE CASE: Solution model being created from scratch (user + reference = complete)
-        elif (
-            self.metadata.schema_ == SchemaCompleteness.complete
-            and self.metadata.data_model_type == DataModelType.solution
-        ):
-            classes += cast(InformationRules, self.rules.reference).model_copy(deep=True).classes
-
-        # USE CASE: Enterprise model being extended (user + last = complete)
-        elif (
-            self.metadata.schema_ == SchemaCompleteness.extended
-            and self.metadata.data_model_type == DataModelType.enterprise
-        ):
-            classes += cast(InformationRules, self.rules.last).model_copy(deep=True).classes
-
         for class_ in classes:
-            class_subclass_pairs[class_.class_] = []
-            if class_.parent is None:
+            class_parent_pairs[class_.class_] = []
+            if class_.implements is None:
                 continue
-            class_subclass_pairs[class_.class_].extend(class_.parent)
+            class_parent_pairs[class_.class_].extend(class_.implements)
 
-        return class_subclass_pairs
+        return class_parent_pairs
 
     def _namespaces_reassigned(self) -> None:
         prefixes = self.rules.prefixes.copy()

cognite/neat/_rules/models/mapping/__init__.py

@@ -1,4 +1,3 @@
-from ._base import Mapping, MappingList, RuleMapping
-from ._classic2core import create_classic_to_core_mapping
+from ._classic2core import load_classic_to_core_mapping
 
-__all__ = ["RuleMapping", "Mapping", "MappingList", "create_classic_to_core_mapping"]
+__all__ = ["load_classic_to_core_mapping"]

cognite/neat/_rules/models/mapping/_classic2core.py

@@ -1,150 +1,40 @@
-from ._base import RuleMapping
+from functools import lru_cache
+from pathlib import Path
 
+import yaml
 
-def create_classic_to_core_mapping() -> RuleMapping:
-    return RuleMapping.model_validate(_CLASSIC_TO_CORE)
+from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._rules._shared import ReadRules
+from cognite.neat._rules.models.dms import DMSInputRules, DMSRules
 
+_CLASSIC_TO_CORE_MAPPING = Path(__file__).resolve().parent / "_classic2core.yaml"
 
-_CLASSIC_TO_CORE = {
-    "classes": [
-        {"destination": {"Class": "cdf_cdm:CogniteAsset"}, "source": {"Class": "classic:Asset"}},
-        {"destination": {"Class": "cdf_cdm:DataProduct"}, "source": {"Class": "classic:DataSet"}},
-        {"destination": {"Class": "cdf_cdm:CogniteActivity"}, "source": {"Class": "classic:Event"}},
-        {"destination": {"Class": "cdf_cdm:CogniteFile"}, "source": {"Class": "classic:File"}},
-        {"destination": {"Class": "cdf_cdm:CogniteTimeSeries"}, "source": {"Class": "classic:TimeSeries"}},
-    ],
-    "properties": [
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "externalId"},
-            "source": {"Class": "classic:Asset", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "name"},
-            "source": {"Class": "classic:Asset", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "parent"},
-            "source": {"Class": "classic:Asset", "Property": "parentId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "description"},
-            "source": {"Class": "classic:Asset", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:Asset", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteAsset", "Property": "tag"},
-            "source": {"Class": "classic:Asset", "Property": "labels"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "externalId"},
-            "source": {"Class": "classic:DataSet", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "name"},
-            "source": {"Class": "classic:DataSet", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "description"},
-            "source": {"Class": "classic:DataSet", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "metadata"},
-            "source": {"Class": "classic:DataSet", "Property": "metadata"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:DataProduct", "Property": "writeProtected"},
-            "source": {"Class": "classic:DataSet", "Property": "writeProtected"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "externalId"},
-            "source": {"Class": "classic:Event", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "startTime"},
-            "source": {"Class": "classic:Event", "Property": "startTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSchedulable", "Property": "endTime"},
-            "source": {"Class": "classic:Event", "Property": "endTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "description"},
-            "source": {"Class": "classic:Event", "Property": "description"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteActivity", "Property": "assets"},
-            "source": {"Class": "classic:Event", "Property": "assetIds"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:Event", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "externalId"},
-            "source": {"Class": "classic:File", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "name"},
-            "source": {"Class": "classic:File", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "directory"},
-            "source": {"Class": "classic:File", "Property": "directory"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteSourceSystem", "Property": "name"},
-            "source": {"Class": "classic:File", "Property": "source"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "mimeType"},
-            "source": {"Class": "classic:File", "Property": "mimeType"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "assets"},
-            "source": {"Class": "classic:File", "Property": "assetIds"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceCreatedTime"},
-            "source": {"Class": "classic:File", "Property": "sourceCreatedTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteFile", "Property": "sourceUpdatedTime"},
-            "source": {"Class": "classic:File", "Property": "sourceModifiedTime"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "externalId"},
-            "source": {"Class": "classic:TimeSeries", "Property": "externalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "name"},
-            "source": {"Class": "classic:TimeSeries", "Property": "name"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "type"},
-            "source": {"Class": "classic:TimeSeries", "Property": "isString"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "sourceUnit"},
-            "source": {"Class": "classic:TimeSeries", "Property": "unit"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "unit"},
-            "source": {"Class": "classic:TimeSeries", "Property": "unitExternalId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "assets"},
-            "source": {"Class": "classic:TimeSeries", "Property": "assetId"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "isStep"},
-            "source": {"Class": "classic:TimeSeries", "Property": "isStep"},
-        },
-        {
-            "destination": {"Class": "cdf_cdm:CogniteTimeSeries", "Property": "description"},
-            "source": {"Class": "classic:TimeSeries", "Property": "description"},
-        },
-    ],
-}
+
+@lru_cache(maxsize=1)
+def _read_source_file() -> str:
+    return _CLASSIC_TO_CORE_MAPPING.read_text()
+
+
+def load_classic_to_core_mapping(org_name: str, source_space: str, source_version: str) -> DMSRules:
+    if not org_name:
+        raise NeatValueError("Organization name must be provided.")
+
+    from cognite.neat._rules.importers import YAMLImporter
+    from cognite.neat._rules.transformers import VerifyDMSRules
+
+    raw_str = _read_source_file().replace("Classic", org_name)
+
+    loaded = yaml.safe_load(raw_str)
+    loaded["metadata"]["space"] = source_space
+    loaded["metadata"]["version"] = source_version
+
+    read: ReadRules[DMSInputRules] = YAMLImporter(loaded).to_rules()
+    if not isinstance(read.rules, DMSInputRules):
+        raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
+
+    verified = VerifyDMSRules(errors="raise", post_validate=False).transform(read)
+
+    if verified.rules is None:
+        raise NeatValueError("Failed to verify the rules.")
+
+    return verified.rules