cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl

cognite/neat/_rules/models/dms/_validation.py

@@ -17,7 +17,7 @@ from cognite.neat._client import NeatClient
 from cognite.neat._client.data_classes.data_modeling import ViewApplyDict
 from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT, DMS_VIEW_CONTAINER_SIZE_LIMIT
-from cognite.neat._issues import IssueList, NeatError, NeatIssueList
+from cognite.neat._issues import IssueList, NeatError
 from cognite.neat._issues.errors import (
     CDFMissingClientError,
     PropertyDefinitionDuplicatedError,
@@ -27,10 +27,12 @@ from cognite.neat._issues.errors import (
     ResourceNotFoundError,
     ReversedConnectionNotFeasibleError,
 )
+from cognite.neat._issues.errors._external import CDFMissingResourcesError
 from cognite.neat._issues.warnings import (
     NotSupportedHasDataFilterLimitWarning,
     NotSupportedViewContainerLimitWarning,
     UndefinedViewWarning,
+    user_modeling,
 )
 from cognite.neat._issues.warnings.user_modeling import (
     ContainerPropertyLimitWarning,
@@ -42,6 +44,8 @@ from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
 from cognite.neat._rules.models.entities._single_value import (
     ViewEntity,
 )
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
+from cognite.neat._utils.text import humanize_collection
 
 from ._rules import DMSProperty, DMSRules
 
@@ -54,13 +58,19 @@ class DMSValidation:
     # For example, changing the filter is allowed, but changing the properties is not.
     changeable_view_attributes: ClassVar[set[str]] = {"filter"}
 
-    def __init__(self, rules: DMSRules, client: NeatClient | None = None) -> None:
+    def __init__(
+        self,
+        rules: DMSRules,
+        client: NeatClient | None = None,
+        read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
+    ) -> None:
         self._rules = rules
         self._client = client
         self._metadata = rules.metadata
         self._properties = rules.properties
         self._containers = rules.containers
         self._views = rules.views
+        self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
 
     def imported_views_and_containers_ids(
         self, include_views_with_no_properties: bool = True
@@ -87,7 +97,7 @@ class DMSValidation:
 
         return imported_views, imported_containers
 
-    def validate(self) -> NeatIssueList:
+    def validate(self) -> IssueList:
         imported_views, imported_containers = self.imported_views_and_containers_ids(
             include_views_with_no_properties=False
         )
@@ -106,6 +116,14 @@ class DMSValidation:
                 list(imported_containers), include_connected=True
             )
 
+            missing_views = {view.as_id() for view in imported_views} - {view.as_id() for view in referenced_views}
+            missing_containers = {container.as_id() for container in imported_containers} - {
+                container.as_id() for container in referenced_containers
+            }
+
+            if missing_views or missing_containers:
+                raise CDFMissingResourcesError(resources=f"{missing_views.union(missing_containers)}")
+
         # Setup data structures for validation
         dms_schema = self._rules.as_schema()
         ref_view_by_id = {view.as_id(): view for view in referenced_views}
@@ -123,6 +141,10 @@ class DMSValidation:
         parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
 
         issue_list = IssueList()
+
+        # Validated for duplicated resource
+        issue_list.extend(self._duplicated_resources())
+
         # Neat DMS classes Validation
         # These are errors that can only happen due to the format of the Neat DMS classes
         issue_list.extend(self._validate_raw_filter())
@@ -141,6 +163,91 @@ class DMSValidation:
         )
         issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
         issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
+        issue_list.extend(self._same_space_views_and_data_model())
+        return issue_list
+
+    def _same_space_views_and_data_model(self) -> IssueList:
+        issue_list = IssueList()
+
+        schema = self._rules.as_schema(remove_cdf_spaces=True)
+
+        if schema.data_model and schema.views:
+            data_model_space = schema.data_model.space
+            views_spaces = {view.space for view in schema.views.values()}
+
+            if data_model_space not in views_spaces:
+                issue_list.append(
+                    user_modeling.ViewsAndDataModelNotInSameSpaceWarning(
+                        data_model_space=data_model_space,
+                        views_spaces=humanize_collection(views_spaces),
+                    )
+                )
+
+        return issue_list
+
+    def _duplicated_resources(self) -> IssueList:
+        issue_list = IssueList()
+
+        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
+        views_sheet = self._read_info_by_spreadsheet.get("Views")
+        containers_sheet = self._read_info_by_spreadsheet.get("Containers")
+
+        visited = defaultdict(list)
+        for row_no, property_ in enumerate(self._properties):
+            visited[property_._identifier()].append(
+                properties_sheet.adjusted_row_number(row_no) if properties_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[1],
+                    "property",
+                    (
+                        f"the Properties sheet at row {humanize_collection(rows)} "
+                        "if data model is read from a spreadsheet."
+                    ),
+                )
+            )
+
+        visited = defaultdict(list)
+        for row_no, view in enumerate(self._views):
+            visited[view._identifier()].append(views_sheet.adjusted_row_number(row_no) if views_sheet else row_no + 1)
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[0],
+                    "view",
+                    (f"the Views sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
+                )
+            )
+
+        if self._containers:
+            visited = defaultdict(list)
+            for row_no, container in enumerate(self._containers):
+                visited[container._identifier()].append(
+                    containers_sheet.adjusted_row_number(row_no) if containers_sheet else row_no + 1
+                )
+
+            for identifier, rows in visited.items():
+                if len(rows) == 1:
+                    continue
+                issue_list.append(
+                    ResourceDuplicatedError(
+                        identifier[0],
+                        "container",
+                        (
+                            f"the Containers sheet at row {humanize_collection(rows)} "
+                            "if data model is read from a spreadsheet."
+                        ),
+                    )
+                )
+
         return issue_list
 
     @staticmethod
@@ -212,13 +319,16 @@ class DMSValidation:
         for prop_no, prop in enumerate(self._properties):
             if prop.container and prop.container_property:
                 container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
-
+        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
         errors = IssueList()
         for (container, prop_name), properties in container_properties_by_id.items():
             if len(properties) == 1:
                 continue
             container_id = container.as_id()
+
             row_numbers = {prop_no for prop_no, _ in properties}
+            if properties_sheet:
+                row_numbers = {properties_sheet.adjusted_row_number(row_no) for row_no in row_numbers}
             value_types = {prop.value_type for _, prop in properties if prop.value_type}
             # The container type 'direct' is an exception. On a container the type direct can point to any
             # node. The value type is typically set on the view.
@@ -585,7 +695,7 @@ class DMSValidation:
                         ResourceDuplicatedError(
                             view_id,
                             "view",
-                            repr(model.as_id()),
+                            f"DMS {model.as_id()!r}",
                         )
                     )
 

cognite/neat/_rules/models/entities/_loaders.py

@@ -23,7 +23,7 @@ def load_value_type(
     elif isinstance(raw, str):
         # property holding xsd data type
         # check if it is multi value type
-        if "|" in raw:
+        if "," in raw:
             value_type = MultiValueTypeInfo.load(raw)
             value_type.set_default_prefix(default_prefix)
             return value_type

cognite/neat/_rules/models/entities/_multi_value.py

@@ -18,7 +18,7 @@ class MultiValueTypeInfo(BaseModel):
     types: list[DataType | ClassEntity]
 
     def __str__(self) -> str:
-        return " | ".join([str(t) for t in self.types])
+        return ", ".join([str(t) for t in self.types])
 
     @model_serializer(when_used="unless-none", return_type=str)
     def as_str(self) -> str:
@@ -52,7 +52,7 @@ class MultiValueTypeInfo(BaseModel):
 
     @classmethod
     def _parse(cls, raw: str) -> dict:
-        if not (types := [type_.strip() for type_ in raw.split("|")]):
+        if not (types := [type_.strip() for type_ in raw.split(",")]):
             return {"types": [UnknownEntity()]}
         else:
             return {

cognite/neat/_rules/models/entities/_single_value.py

@@ -35,6 +35,7 @@ else:
 from cognite.neat._rules._constants import (
     ENTITY_PATTERN,
     SPLIT_ON_COMMA_PATTERN,
+    SPLIT_ON_EDGE_ENTITY_ARGS_PATTERN,
     SPLIT_ON_EQUAL_PATTERN,
     EntityTypes,
 )
@@ -135,9 +136,13 @@ class Entity(BaseModel, extra="ignore"):
         if content is None:
             return dict(prefix=prefix, suffix=suffix)
         try:
-            extra_args = dict(
-                SPLIT_ON_EQUAL_PATTERN.split(pair.strip()) for pair in SPLIT_ON_COMMA_PATTERN.split(content)
-            )
+            if cls == EdgeEntity:
+                matches = SPLIT_ON_EDGE_ENTITY_ARGS_PATTERN.findall(content)
+                extra_args = {key: value for key, value in matches}
+            else:
+                extra_args = dict(
+                    SPLIT_ON_EQUAL_PATTERN.split(pair.strip()) for pair in SPLIT_ON_COMMA_PATTERN.split(content)
+                )
         except ValueError:
             raise NeatValueError(f"Invalid {cls.type_.value} entity: {raw!r}") from None
         expected_args = {

cognite/neat/_rules/models/entities/_wrapped.py

@@ -127,8 +127,8 @@ class DMSFilter(WrappedEntity):
                     if isinstance(entry, dict) and "space" in entry and "externalId" in entry
                 ]
             )
-
-        raise ValueError(f"Cannot convert {filter._filter_name} to {cls.__name__}")
+        # fall back to raw filter to preserve the information
+        return RawFilter(filter=json.dumps(dumped))
 
 
 class NodeTypeFilter(DMSFilter):

cognite/neat/_rules/models/information/_rules.py

@@ -8,7 +8,7 @@ from pydantic_core.core_schema import SerializationInfo
 from rdflib import Namespace, URIRef
 
 from cognite.neat._constants import get_default_prefixes_and_namespaces
-from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
+from cognite.neat._issues.errors import PropertyDefinitionError
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
@@ -18,11 +18,6 @@ from cognite.neat._rules.models._base_rules import (
     SheetList,
     SheetRow,
 )
-from cognite.neat._rules.models._rdfpath import (
-    RDFPath,
-    TransformationRuleType,
-    parse_rule,
-)
 from cognite.neat._rules.models._types import (
     ClassEntityType,
     InformationPropertyType,
@@ -78,7 +73,11 @@ class InformationClass(SheetRow):
         default=None,
         description="List of classes (comma separated) that the current class implements (parents).",
     )
-
+    instance_source: URIRefType | None = Field(
+        alias="Instance Source",
+        default=None,
+        description="The link to to the rdf.type that have the instances for this class.",
+    )
     physical: URIRefType | None = Field(
         None,
         description="Link to the class representation in the physical data model aspect",
@@ -153,11 +152,10 @@ class InformationProperty(SheetRow):
         "which means that the property can hold any number of values (listable).",
     )
     default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
-    instance_source: RDFPath | None = Field(
+    instance_source: list[URIRefType] | None = Field(
         alias="Instance Source",
         default=None,
-        description="The link to to the instance property for the model. "
-        "The rule is provided in a RDFPath query syntax which is converted to downstream solution query (e.g. SPARQL).",
+        description="The URIRef(s) in the graph to get the value of the property.",
     )
     inherited: bool = Field(
         default=False,
@@ -182,13 +180,10 @@ class InformationProperty(SheetRow):
         return value
 
     @field_validator("instance_source", mode="before")
-    def generate_rdfpath(cls, value: str | RDFPath | None) -> RDFPath | None:
-        if value is None or isinstance(value, RDFPath):
-            return value
-        elif isinstance(value, str):
-            return parse_rule(value, TransformationRuleType.rdfpath)
-        else:
-            raise NeatValueError(f"Invalid RDF Path: {value!s}")
+    def split_on_comma(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return [v.strip() for v in value.split(",")]
+        return value
 
     @model_validator(mode="after")
     def set_type_for_default(self):
@@ -214,6 +209,12 @@ class InformationProperty(SheetRow):
                     ) from None
         return self
 
+    @field_serializer("instance_source", when_used="unless-none")
+    def serialize_instance_source(self, value: list[URIRefType] | None) -> str | None:
+        if value is None:
+            return None
+        return ",".join(str(v) for v in value)
+
     @field_serializer("max_count", when_used="json-unless-none")
     def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
         if isinstance(value, float) and math.isinf(value):

cognite/neat/_rules/models/information/_rules_input.py

@@ -84,7 +84,7 @@ class InformationInputProperty(InputComponent[InformationProperty]):
     min_count: int | None = None
     max_count: int | float | None = None
     default: Any | None = None
-    instance_source: str | None = None
+    instance_source: str | list[str] | None = None
     # Only used internally
     inherited: bool = False
     neatId: str | URIRef | None = None
@@ -110,6 +110,7 @@ class InformationInputClass(InputComponent[InformationClass]):
     name: str | None = None
     description: str | None = None
     implements: str | list[ClassEntity] | None = None
+    instance_source: str | None = None
     neatId: str | URIRef | None = None
     # linking
     physical: str | URIRef | None = None
@@ -127,6 +128,7 @@ class InformationInputClass(InputComponent[InformationClass]):
         output = super().dump()
         parent: list[ClassEntity] | None = None
         if isinstance(self.implements, str):
+            self.implements = self.implements.strip()
             parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
         elif isinstance(self.implements, list):
             parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]

cognite/neat/_rules/models/information/_validation.py

@@ -1,9 +1,12 @@
 import itertools
-from collections import Counter
+from collections import Counter, defaultdict
 
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._issues.errors._resources import ResourceNotDefinedError
+from cognite.neat._issues.errors._resources import (
+    ResourceDuplicatedError,
+    ResourceNotDefinedError,
+)
 from cognite.neat._issues.warnings._models import UndefinedClassWarning
 from cognite.neat._issues.warnings._resources import (
     ResourceNotDefinedWarning,
@@ -12,6 +15,8 @@ from cognite.neat._issues.warnings._resources import (
 from cognite.neat._rules._constants import PATTERNS, EntityTypes
 from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
 from cognite.neat._rules.models.entities._multi_value import MultiValueTypeInfo
+from cognite.neat._utils.spreadsheet import SpreadsheetRead
+from cognite.neat._utils.text import humanize_collection
 
 from ._rules import InformationRules
 
@@ -20,14 +25,16 @@ class InformationValidation:
     """This class does all the validation of the Information rules that have dependencies
     between components."""
 
-    def __init__(self, rules: InformationRules):
+    def __init__(self, rules: InformationRules, read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None):
         self.rules = rules
-        self.metadata = rules.metadata
-        self.properties = rules.properties
-        self.classes = rules.classes
+        self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
+        self._metadata = rules.metadata
+        self._properties = rules.properties
+        self._classes = rules.classes
         self.issue_list = IssueList()
 
     def validate(self) -> IssueList:
+        self._duplicated_resources()
         self._namespaces_reassigned()
         self._classes_without_properties()
         self._undefined_classes()
@@ -38,9 +45,51 @@ class InformationValidation:
 
         return self.issue_list
 
+    def _duplicated_resources(self) -> None:
+        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
+        classes_sheet = self._read_info_by_spreadsheet.get("Classes")
+
+        visited = defaultdict(list)
+        for row_no, property_ in enumerate(self._properties):
+            visited[property_._identifier()].append(
+                properties_sheet.adjusted_row_number(row_no) if properties_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            self.issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[1],
+                    "property",
+                    (
+                        "the Properties sheet at row "
+                        f"{humanize_collection(rows)}"
+                        " if data model is read from a spreadsheet."
+                    ),
+                )
+            )
+
+        visited = defaultdict(list)
+        for row_no, class_ in enumerate(self._classes):
+            visited[class_._identifier()].append(
+                classes_sheet.adjusted_row_number(row_no) if classes_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            self.issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[0],
+                    "class",
+                    (f"the Classes sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
+                )
+            )
+
     def _classes_without_properties(self) -> None:
-        defined_classes = {class_.class_ for class_ in self.classes}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        referred_classes = {property_.class_ for property_ in self._properties}
         class_parent_pairs = self._class_parent_pairs()
 
         if classes_without_properties := defined_classes.difference(referred_classes):
@@ -48,7 +97,7 @@ class InformationValidation:
                 # USE CASE: class has no direct properties and no parents with properties
                 # and it is a class in the prefix of data model, as long as it is in the
                 # same prefix, meaning same space
-                if not class_parent_pairs[class_] and class_.prefix == self.metadata.prefix:
+                if not class_parent_pairs[class_] and class_.prefix == self._metadata.prefix:
                     self.issue_list.append(
                         ResourceNotDefinedWarning(
                             resource_type="class",
@@ -58,8 +107,8 @@ class InformationValidation:
                     )
 
     def _undefined_classes(self) -> None:
-        defined_classes = {class_.class_ for class_ in self.classes}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        referred_classes = {property_.class_ for property_ in self._properties}
 
         if undefined_classes := referred_classes.difference(defined_classes):
             for class_ in undefined_classes:
@@ -79,7 +128,7 @@ class InformationValidation:
 
         if undefined_parents := parents.difference(classes):
             for parent in undefined_parents:
-                if parent.prefix != self.metadata.prefix:
+                if parent.prefix != self._metadata.prefix:
                     self.issue_list.append(UndefinedClassWarning(class_id=str(parent)))
                 else:
                     self.issue_list.append(
@@ -92,8 +141,8 @@ class InformationValidation:
 
     def _referenced_classes_exist(self) -> None:
         # needs to be complete for this validation to pass
-        defined_classes = {class_.class_ for class_ in self.classes}
-        classes_with_explicit_properties = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        classes_with_explicit_properties = {property_.class_ for property_ in self._properties}
 
         # USE CASE: models are complete
         if missing_classes := classes_with_explicit_properties.difference(defined_classes):
@@ -108,7 +157,7 @@ class InformationValidation:
 
     def _referenced_value_types_exist(self) -> None:
         # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
-        defined_classes = {class_.class_ for class_ in self.classes} | {UnknownEntity()}
+        defined_classes = {class_.class_ for class_ in self._classes} | {UnknownEntity()}
         referred_object_types = {
             property_.value_type
             for property_ in self.rules.properties
@@ -129,7 +178,7 @@ class InformationValidation:
     def _regex_compliance_with_dms(self) -> None:
         """Check regex compliance with DMS of properties, classes and value types."""
 
-        for prop_ in self.properties:
+        for prop_ in self._properties:
             if not PATTERNS.dms_property_id_compliance.match(prop_.property_):
                 self.issue_list.append(
                     ResourceRegexViolationWarning(
@@ -179,7 +228,7 @@ class InformationValidation:
                             )
                         )
 
-        for class_ in self.classes:
+        for class_ in self._classes:
             if not PATTERNS.view_id_compliance.match(class_.class_.suffix):
                 self.issue_list.append(
                     ResourceRegexViolationWarning(

cognite/neat/_rules/transformers/__init__.py

@@ -14,11 +14,14 @@ from ._converters import (
     PrefixEntities,
     SetIDDMSModel,
     StandardizeNaming,
+    StandardizeSpaceAndVersion,
+    SubsetDMSRules,
+    SubsetInformationRules,
     ToCompliantEntities,
     ToDataProductModel,
+    ToDMSCompliantEntities,
     ToEnterpriseModel,
     ToExtensionModel,
-    ToInformationCompliantEntities,
     ToSolutionModel,
 )
 from ._mapping import AsParentPropertyId, MapOneToOne, RuleMapper
@@ -43,11 +46,14 @@ __all__ = [
     "RulesTransformer",
     "SetIDDMSModel",
     "StandardizeNaming",
+    "StandardizeSpaceAndVersion",
+    "SubsetDMSRules",
+    "SubsetInformationRules",
     "ToCompliantEntities",
+    "ToDMSCompliantEntities",
     "ToDataProductModel",
     "ToEnterpriseModel",
     "ToExtensionModel",
-    "ToInformationCompliantEntities",
     "ToSolutionModel",
     "VerifiedRulesTransformer",
     "VerifyAnyRules",