cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_inference2rules.py

@@ -16,6 +16,7 @@ from cognite.neat._rules.models.information import (
     InformationMetadata,
 )
 from cognite.neat._store import NeatGraphStore
+from cognite.neat._store._provenance import INSTANCES_ENTITY
 from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
 
 from ._base import DEFAULT_NON_EXISTING_NODE_TYPE, BaseRDFImporter
@@ -94,6 +95,7 @@ class InferenceImporter(BaseRDFImporter):
         max_number_of_instance: int = -1,
         non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
         language: str = "en",
+        source_name: str = "Unknown",
     ) -> "InferenceImporter":
         return super().from_file(
             filepath,
@@ -101,6 +103,7 @@ class InferenceImporter(BaseRDFImporter):
             max_number_of_instance,
             non_existing_node_type,
             language,
+            source_name=source_name,
         )
 
     @classmethod
@@ -283,3 +286,7 @@ class InferenceImporter(BaseRDFImporter):
             updated=now,
             description="Inferred model from knowledge graph",
         )
+
+    @property
+    def source_uri(self) -> URIRef:
+        return INSTANCES_ENTITY.id_

cognite/neat/_rules/importers/_rdf/_owl2rules.py

@@ -78,3 +78,7 @@ class OWLImporter(BaseRDFImporter):
         }
 
         return components
+
+    @property
+    def description(self) -> str:
+        return f"Ontology {self.source_name} read as unverified data model"

cognite/neat/_rules/importers/_spreadsheet2rules.py

@@ -11,9 +11,9 @@ from typing import Literal, cast
 import pandas as pd
 from cognite.client.utils._importing import local_import
 from pandas import ExcelFile
-from rdflib import Namespace
+from rdflib import Namespace, URIRef
 
-from cognite.neat._issues import IssueList
+from cognite.neat._issues import IssueList, MultiValueError
 from cognite.neat._issues.errors import (
     FileMissingRequiredFieldError,
     FileNotFoundNeatError,
@@ -253,18 +253,19 @@ class ExcelImporter(BaseImporter[T_InputRules]):
     def to_rules(self) -> ReadRules[T_InputRules]:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
         if not self.filepath.exists():
-            issue_list.append(FileNotFoundNeatError(self.filepath))
-            return ReadRules(None, issue_list, {})
+            raise FileNotFoundNeatError(self.filepath)
 
         with pd.ExcelFile(self.filepath) as excel_file:
             user_reader = SpreadsheetReader(issue_list)
 
             user_read = user_reader.read(excel_file, self.filepath)
-            if user_read is None or issue_list.has_errors:
-                return ReadRules(None, issue_list, {})
 
+        issue_list.trigger_warnings()
         if issue_list.has_errors:
-            return ReadRules(None, issue_list, {})
+            raise MultiValueError(issue_list.errors)
+
+        if user_read is None:
+            return ReadRules(None, {})
 
         sheets = user_read.sheets
         original_role = user_read.role
@@ -272,7 +273,15 @@ class ExcelImporter(BaseImporter[T_InputRules]):
 
         rules_cls = INPUT_RULES_BY_ROLE[original_role]
         rules = cast(T_InputRules, rules_cls.load(sheets))
-        return ReadRules(rules, issue_list, {"read_info_by_sheet": read_info_by_sheet})
+        return ReadRules(rules, {"read_info_by_sheet": read_info_by_sheet})
+
+    @property
+    def description(self) -> str:
+        return f"Excel file {self.filepath.name} read as unverified data model"
+
+    @property
+    def source_uri(self) -> URIRef:
+        return URIRef(f"file://{self.filepath.name}")
 
 
 class GoogleSheetImporter(BaseImporter[T_InputRules]):

cognite/neat/_rules/importers/_yaml2rules.py

@@ -3,7 +3,7 @@ from typing import Any, cast
 
 import yaml
 
-from cognite.neat._issues import IssueList, NeatIssue
+from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
 from cognite.neat._issues.errors import (
     FileMissingRequiredFieldError,
     FileNotAFileError,
@@ -36,24 +36,32 @@ class YAMLImporter(BaseImporter[T_InputRules]):
         raw_data: dict[str, Any],
         read_issues: list[NeatIssue] | None = None,
         filepaths: list[Path] | None = None,
+        source_name: str = "Unknown",
     ) -> None:
         self.raw_data = raw_data
         self._read_issues = IssueList(read_issues)
         self._filepaths = filepaths
+        self._source_name = source_name
+
+    @property
+    def description(self) -> str:
+        return f"YAML file {self._source_name} read as unverified data model"
 
     @classmethod
-    def from_file(cls, filepath: Path):
+    def from_file(cls, filepath: Path, source_name: str = "Unknown") -> "YAMLImporter":
         if not filepath.exists():
             return cls({}, [FileNotFoundNeatError(filepath)])
         elif not filepath.is_file():
             return cls({}, [FileNotAFileError(filepath)])
         elif filepath.suffix not in [".yaml", ".yml"]:
             return cls({}, [FileTypeUnexpectedError(filepath, frozenset([".yaml", ".yml"]))])
-        return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
+        return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath], source_name=source_name)
 
     def to_rules(self) -> ReadRules[T_InputRules]:
         if self._read_issues.has_errors or not self.raw_data:
-            return ReadRules(None, self._read_issues, {})
+            self._read_issues.trigger_warnings()
+            raise MultiValueError(self._read_issues.errors)
+
         issue_list = IssueList(title="YAML Importer", issues=self._read_issues)
 
         if not self._filepaths:
@@ -69,13 +77,15 @@ class YAMLImporter(BaseImporter[T_InputRules]):
 
         if "metadata" not in self.raw_data:
             self._read_issues.append(FileMissingRequiredFieldError(metadata_file, "section", "metadata"))
-            return ReadRules(None, self._read_issues, {})
+            issue_list.trigger_warnings()
+            raise MultiValueError(self._read_issues.errors)
 
         metadata = self.raw_data["metadata"]
 
         if "role" not in metadata:
             self._read_issues.append(FileMissingRequiredFieldError(metadata, "metadata", "role"))
-            return ReadRules(None, self._read_issues, {})
+            issue_list.trigger_warnings()
+            raise MultiValueError(self._read_issues.errors)
 
         role_input = RoleTypes(metadata["role"])
         role_enum = RoleTypes(role_input)
@@ -83,4 +93,8 @@ class YAMLImporter(BaseImporter[T_InputRules]):
 
         rules = cast(T_InputRules, rules_cls.load(self.raw_data))
 
-        return ReadRules(rules, issue_list, {})
+        issue_list.trigger_warnings()
+        if self._read_issues.has_errors:
+            raise MultiValueError(self._read_issues.errors)
+
+        return ReadRules[T_InputRules](rules, {})

cognite/neat/_rules/models/_base_input.py

@@ -110,7 +110,7 @@ class InputRules(Generic[T_BaseRules], ABC):
     def _dataclass_fields(self) -> list[Field]:
         return list(fields(self))
 
-    def as_rules(self) -> T_BaseRules:
+    def as_verified_rules(self) -> T_BaseRules:
         cls_ = self._get_verified_cls()
         return cls_.model_validate(self.dump())
 

cognite/neat/_rules/models/_base_rules.py

@@ -50,6 +50,7 @@ from cognite.neat._rules.models._types import (
 )
 from cognite.neat._rules.models.data_types import DataType
 from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
+from cognite.neat._utils.rdf_ import uri_display_name
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -285,6 +286,10 @@ class BaseRules(SchemaModel, ABC):
             ]
         return headers_by_sheet
 
+    @property
+    def display_name(self):
+        return uri_display_name(self.metadata.identifier)
+
     def dump(
         self,
         entities_exclude_defaults: bool = True,
@@ -393,7 +398,10 @@ class SheetList(list, MutableSequence[T_SheetRow]):
 
     def _repr_html_(self) -> str:
         """Returns HTML representation of ResourceDict."""
-        return self.to_pandas(drop_na_columns=True)._repr_html_()  # type: ignore[operator]
+        df = self.to_pandas(drop_na_columns=True)
+        if "neatId" in df.columns:
+            df = df.drop(columns=["neatId"])
+        return df._repr_html_()  # type: ignore[operator]
 
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[T_SheetRow]:

cognite/neat/_rules/models/dms/_rules.py

@@ -447,6 +447,10 @@ class DMSRules(BaseRules):
 
         return _DMSExporter(self, instance_space, remove_cdf_spaces=remove_cdf_spaces).to_schema()
 
+    @classmethod
+    def display_type_name(cls) -> str:
+        return "VerifiedDMSModel"
+
     def _repr_html_(self) -> str:
         summary = {
             "aspect": self.metadata.aspect,

cognite/neat/_rules/models/dms/_rules_input.py

@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
     load_connection,
     load_dms_value_type,
 )
+from cognite.neat._utils.rdf_ import uri_display_name
 
 from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
 
@@ -289,6 +290,14 @@ class DMSInputRules(InputRules[DMSRules]):
             "Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
         }
 
+    @classmethod
+    def display_type_name(cls) -> str:
+        return "UnverifiedDMSModel"
+
+    @property
+    def display_name(self):
+        return uri_display_name(self.metadata.identifier)
+
     def _repr_html_(self) -> str:
         summary = {
             "type": "Physical Data Model",

cognite/neat/_rules/models/entities/_wrapped.py

@@ -9,7 +9,7 @@ from cognite.client import data_modeling as dm
 from cognite.client.data_classes.data_modeling import ContainerId, NodeId
 from pydantic import BaseModel, model_serializer, model_validator
 
-from ._single_value import ContainerEntity, DMSNodeEntity, Entity
+from ._single_value import ContainerEntity, DMSNodeEntity, Entity, ViewEntity
 
 
 @total_ordering
@@ -121,6 +121,8 @@ class DMSFilter(WrappedEntity):
             return HasDataFilter(
                 inner=[
                     ContainerEntity(space=entry["space"], externalId=entry["externalId"])
+                    if entry["type"] == "container"
+                    else ViewEntity(space=entry["space"], externalId=entry["externalId"], version=entry["version"])
                     for entry in body
                     if isinstance(entry, dict) and "space" in entry and "externalId" in entry
                 ]
@@ -161,21 +163,24 @@ class NodeTypeFilter(DMSFilter):
 
 class HasDataFilter(DMSFilter):
     name: ClassVar[str] = "hasData"
-    _inner_cls: ClassVar[type[ContainerEntity]] = ContainerEntity
-    inner: list[ContainerEntity] | None = None  # type: ignore[assignment]
+    _inner_cls: ClassVar[type[ContainerEntity | ViewEntity]] = ContainerEntity
+    inner: list[ContainerEntity | ViewEntity] | None = None  # type: ignore[assignment]
 
     def as_dms_filter(self, default: Collection[ContainerId] | None = None) -> dm.Filter:
         containers: list[ContainerId]
         if self.inner:
-            containers = [container.as_id() for container in self.inner]
+            containers = [item.as_id() for item in self.inner if isinstance(item, ContainerEntity)]
+            views = [item.as_id() for item in self.inner if isinstance(item, ViewEntity)]
         elif default:
             containers = list(default)
+            views = []
         else:
             raise ValueError("Empty hasData filter, please provide a default containers.")
 
         return dm.filters.HasData(
             # Sorting to ensure deterministic order
-            containers=sorted(containers, key=lambda container: container.as_tuple())  # type: ignore[union-attr]
+            containers=sorted(containers, key=lambda container: container.as_tuple()),  # type: ignore[union-attr]
+            views=sorted(views, key=lambda view: view.as_tuple()),  # type: ignore[union-attr]
         )
 
 

cognite/neat/_rules/models/information/_rules.py

@@ -272,6 +272,10 @@ class InformationRules(BaseRules):
 
         return _InformationRulesConverter(self).as_dms_rules()
 
+    @classmethod
+    def display_type_name(cls) -> str:
+        return "VerifiedInformationModel"
+
     def _repr_html_(self) -> str:
         summary = {
             "type": "Logical Data Model",

cognite/neat/_rules/models/information/_rules_input.py

@@ -14,6 +14,7 @@ from cognite.neat._rules.models.entities import (
     UnknownEntity,
     load_value_type,
 )
+from cognite.neat._utils.rdf_ import uri_display_name
 
 from ._rules import (
     InformationClass,
@@ -150,6 +151,14 @@ class InformationInputRules(InputRules[InformationRules]):
             Prefixes=self.prefixes,
         )
 
+    @classmethod
+    def display_type_name(cls) -> str:
+        return "UnverifiedInformationModel"
+
+    @property
+    def display_name(self):
+        return uri_display_name(self.metadata.identifier)
+
     def _repr_html_(self) -> str:
         summary = {
             "type": "Logical Data Model",

cognite/neat/_rules/models/mapping/_classic2core.py

@@ -32,9 +32,6 @@ def load_classic_to_core_mapping(org_name: str, source_space: str, source_versio
     if not isinstance(read.rules, DMSInputRules):
         raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
 
-    verified = VerifyDMSRules(errors="raise", validate=False).transform(read)
+    verified = VerifyDMSRules(validate=False).transform(read)
 
-    if verified.rules is None:
-        raise NeatValueError("Failed to verify the rules.")
-
-    return verified.rules
+    return verified