cognite-neat 0.110.0__py3-none-any.whl → 0.111.0__py3-none-any.whl

cognite/neat/_rules/models/data_types.py

@@ -130,6 +130,13 @@ class DataType(BaseModel):
     def as_xml_uri_ref(cls) -> URIRef:
         return XML_SCHEMA_NAMESPACE[cls.xsd]
 
+    @classmethod
+    def convert_value(cls, value: Any) -> Any:
+        if cls != Boolean:
+            return cls.python(value)
+        else:
+            return value.strip().lower() in {"true", "1", "yes"}
+
 
 class Boolean(DataType):
     python = bool

cognite/neat/_rules/models/dms/_exporter.py

@@ -99,6 +99,7 @@ class _DMSExporter:
             key=lambda x: x.as_tuple(),  # type: ignore[union-attr]
         )
         spaces = self._create_spaces(rules.metadata, containers, views, data_model)
+
         return DMSSchema(
             spaces=spaces,
             data_model=data_model,
@@ -114,14 +115,14 @@ class _DMSExporter:
         views: ViewApplyDict,
         data_model: dm.DataModelApply,
     ) -> SpaceApplyDict:
-        used_spaces = {container.space for container in containers.values()} | {view.space for view in views.values()}
-        if len(used_spaces) == 1:
-            # We skip the default space and only use this space for the data model
-            data_model.space = used_spaces.pop()
-            spaces = SpaceApplyDict([dm.SpaceApply(space=data_model.space)])
-        else:
-            used_spaces.add(metadata.space)
-            spaces = SpaceApplyDict([dm.SpaceApply(space=space) for space in used_spaces])
+        used_spaces = (
+            {container.space for container in containers.values()}
+            | {view.space for view in views.values()}
+            | {data_model.space}
+            | {metadata.space}
+        )
+
+        spaces = SpaceApplyDict([dm.SpaceApply(space=space) for space in used_spaces])
         if self.instance_space and self.instance_space not in spaces:
             spaces[self.instance_space] = dm.SpaceApply(space=self.instance_space, name=self.instance_space)
         return spaces

cognite/neat/_rules/models/dms/_rules.py

@@ -1,3 +1,4 @@
+import warnings
 from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar, Literal
 
@@ -8,6 +9,7 @@ from pydantic_core.core_schema import SerializationInfo, ValidationInfo
 
 from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._issues.errors import NeatValueError
+from cognite.neat._issues.warnings._general import NeatValueWarning
 from cognite.neat._rules.models._base_rules import (
     BaseMetadata,
     BaseRules,
@@ -116,7 +118,7 @@ class DMSProperty(SheetRow):
         description="Used to indicate whether the property holds single or multiple values (list). "
         "Only applies to primitive types.",
     )
-    default: str | int | dict | None = Field(
+    default: bool | str | int | float | dict | None = Field(
         None, alias="Default", description="Specifies default value for the property."
     )
     container: ContainerEntityType | None = Field(
@@ -168,6 +170,29 @@ class DMSProperty(SheetRow):
             raise ValueError(f"Reverse connection must have a value type that points to a view, got {value}")
         return value
 
+    @field_validator("default", mode="after")
+    def set_proper_type_on_default(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+        value_type = info.data.get("value_type")
+        if not isinstance(value_type, DataType):
+            warnings.filterwarnings("default")
+            warnings.warn(
+                NeatValueWarning(f"Default value {value} set to connection {value_type} will be ignored"),
+                stacklevel=2,
+            )
+            return None
+        else:
+            try:
+                return value_type.convert_value(value)
+            except ValueError:
+                warnings.filterwarnings("default")
+                warnings.warn(
+                    NeatValueWarning(f"Could not convert {value} to {value_type}"),
+                    stacklevel=2,
+                )
+                return None
+
     @field_validator("container", "container_property", mode="after")
     def container_set_correctly(cls, value: Any, info: ValidationInfo) -> Any:
         if (connection := info.data.get("connection")) is None:

cognite/neat/_rules/models/dms/_rules_input.py

@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
     load_connection,
     load_dms_value_type,
 )
+from cognite.neat._rules.models.entities._wrapped import DMSFilter
 from cognite.neat._utils.rdf_ import uri_display_name
 
 from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
@@ -69,7 +70,7 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
     def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
         if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
             creator = description_match.group(1).split(", ")
-            description = description_raw.replace(description_match.string, "").strip() or None
+            description = description_raw.replace(description_match[0], "").strip() or None
         elif description_raw:
             creator = ["MISSING"]
             description = description_raw
@@ -211,6 +212,8 @@ class DMSInputView(InputComponent[DMSView]):
         return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
 
     def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
+        self.implements = self.implements.strip() if self.implements else None
+
         return (
             [
                 ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
@@ -234,6 +237,7 @@ class DMSInputView(InputComponent[DMSView]):
             implements=", ".join([str(ViewEntity.from_id(parent, _DEFAULT_VERSION)) for parent in view.implements])
             or None,
             in_model=in_model,
+            filter_=(str(DMSFilter.from_dms_filter(view.filter)) if view.filter else None),
         )
 
 

cognite/neat/_rules/models/dms/_validation.py

@@ -27,10 +27,12 @@ from cognite.neat._issues.errors import (
     ResourceNotFoundError,
     ReversedConnectionNotFeasibleError,
 )
+from cognite.neat._issues.errors._external import CDFMissingResourcesError
 from cognite.neat._issues.warnings import (
     NotSupportedHasDataFilterLimitWarning,
     NotSupportedViewContainerLimitWarning,
     UndefinedViewWarning,
+    user_modeling,
 )
 from cognite.neat._issues.warnings.user_modeling import (
     ContainerPropertyLimitWarning,
@@ -43,6 +45,7 @@ from cognite.neat._rules.models.entities._single_value import (
     ViewEntity,
 )
 from cognite.neat._utils.spreadsheet import SpreadsheetRead
+from cognite.neat._utils.text import humanize_collection
 
 from ._rules import DMSProperty, DMSRules
 
@@ -113,6 +116,14 @@ class DMSValidation:
                 list(imported_containers), include_connected=True
             )
 
+            missing_views = {view.as_id() for view in imported_views} - {view.as_id() for view in referenced_views}
+            missing_containers = {container.as_id() for container in imported_containers} - {
+                container.as_id() for container in referenced_containers
+            }
+
+            if missing_views or missing_containers:
+                raise CDFMissingResourcesError(resources=f"{missing_views.union(missing_containers)}")
+
         # Setup data structures for validation
         dms_schema = self._rules.as_schema()
         ref_view_by_id = {view.as_id(): view for view in referenced_views}
@@ -130,6 +141,10 @@ class DMSValidation:
         parents_view_ids_by_child_id = self._parent_view_ids_by_child_id(all_views_by_id)
 
         issue_list = IssueList()
+
+        # Validated for duplicated resource
+        issue_list.extend(self._duplicated_resources())
+
         # Neat DMS classes Validation
         # These are errors that can only happen due to the format of the Neat DMS classes
         issue_list.extend(self._validate_raw_filter())
@@ -148,6 +163,91 @@ class DMSValidation:
         )
         issue_list.extend(self._validate_schema(dms_schema, all_views_by_id, all_containers_by_id))
         issue_list.extend(self._validate_referenced_container_limits(dms_schema.views, view_properties_by_id))
+        issue_list.extend(self._same_space_views_and_data_model())
+        return issue_list
+
+    def _same_space_views_and_data_model(self) -> IssueList:
+        issue_list = IssueList()
+
+        schema = self._rules.as_schema(remove_cdf_spaces=True)
+
+        if schema.data_model and schema.views:
+            data_model_space = schema.data_model.space
+            views_spaces = {view.space for view in schema.views.values()}
+
+            if data_model_space not in views_spaces:
+                issue_list.append(
+                    user_modeling.ViewsAndDataModelNotInSameSpaceWarning(
+                        data_model_space=data_model_space,
+                        views_spaces=humanize_collection(views_spaces),
+                    )
+                )
+
+        return issue_list
+
+    def _duplicated_resources(self) -> IssueList:
+        issue_list = IssueList()
+
+        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
+        views_sheet = self._read_info_by_spreadsheet.get("Views")
+        containers_sheet = self._read_info_by_spreadsheet.get("Containers")
+
+        visited = defaultdict(list)
+        for row_no, property_ in enumerate(self._properties):
+            visited[property_._identifier()].append(
+                properties_sheet.adjusted_row_number(row_no) if properties_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[1],
+                    "property",
+                    (
+                        f"the Properties sheet at row {humanize_collection(rows)} "
+                        "if data model is read from a spreadsheet."
+                    ),
+                )
+            )
+
+        visited = defaultdict(list)
+        for row_no, view in enumerate(self._views):
+            visited[view._identifier()].append(views_sheet.adjusted_row_number(row_no) if views_sheet else row_no + 1)
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[0],
+                    "view",
+                    (f"the Views sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
+                )
+            )
+
+        if self._containers:
+            visited = defaultdict(list)
+            for row_no, container in enumerate(self._containers):
+                visited[container._identifier()].append(
+                    containers_sheet.adjusted_row_number(row_no) if containers_sheet else row_no + 1
+                )
+
+            for identifier, rows in visited.items():
+                if len(rows) == 1:
+                    continue
+                issue_list.append(
+                    ResourceDuplicatedError(
+                        identifier[0],
+                        "container",
+                        (
+                            f"the Containers sheet at row {humanize_collection(rows)} "
+                            "if data model is read from a spreadsheet."
+                        ),
+                    )
+                )
+
         return issue_list
 
     @staticmethod
@@ -595,7 +695,7 @@ class DMSValidation:
                         ResourceDuplicatedError(
                             view_id,
                             "view",
-                            repr(model.as_id()),
+                            f"DMS {model.as_id()!r}",
                         )
                     )
 

cognite/neat/_rules/models/entities/_single_value.py

@@ -35,6 +35,7 @@ else:
 from cognite.neat._rules._constants import (
     ENTITY_PATTERN,
     SPLIT_ON_COMMA_PATTERN,
+    SPLIT_ON_EDGE_ENTITY_ARGS_PATTERN,
     SPLIT_ON_EQUAL_PATTERN,
     EntityTypes,
 )
@@ -135,9 +136,13 @@ class Entity(BaseModel, extra="ignore"):
         if content is None:
             return dict(prefix=prefix, suffix=suffix)
         try:
-            extra_args = dict(
-                SPLIT_ON_EQUAL_PATTERN.split(pair.strip()) for pair in SPLIT_ON_COMMA_PATTERN.split(content)
-            )
+            if cls == EdgeEntity:
+                matches = SPLIT_ON_EDGE_ENTITY_ARGS_PATTERN.findall(content)
+                extra_args = {key: value for key, value in matches}
+            else:
+                extra_args = dict(
+                    SPLIT_ON_EQUAL_PATTERN.split(pair.strip()) for pair in SPLIT_ON_COMMA_PATTERN.split(content)
+                )
         except ValueError:
             raise NeatValueError(f"Invalid {cls.type_.value} entity: {raw!r}") from None
         expected_args = {

cognite/neat/_rules/models/entities/_wrapped.py

@@ -127,8 +127,8 @@ class DMSFilter(WrappedEntity):
                     if isinstance(entry, dict) and "space" in entry and "externalId" in entry
                 ]
             )
-
-        raise ValueError(f"Cannot convert {filter._filter_name} to {cls.__name__}")
+        # fall back to raw filter to preserve the information
+        return RawFilter(filter=json.dumps(dumped))
 
 
 class NodeTypeFilter(DMSFilter):

cognite/neat/_rules/models/information/_rules_input.py

@@ -128,6 +128,7 @@ class InformationInputClass(InputComponent[InformationClass]):
         output = super().dump()
         parent: list[ClassEntity] | None = None
         if isinstance(self.implements, str):
+            self.implements = self.implements.strip()
             parent = [ClassEntity.load(parent, prefix=default_prefix) for parent in self.implements.split(",")]
         elif isinstance(self.implements, list):
             parent = [ClassEntity.load(parent_, prefix=default_prefix) for parent_ in self.implements]

cognite/neat/_rules/models/information/_validation.py

@@ -1,9 +1,12 @@
 import itertools
-from collections import Counter
+from collections import Counter, defaultdict
 
 from cognite.neat._issues import IssueList
 from cognite.neat._issues.errors import NeatValueError
-from cognite.neat._issues.errors._resources import ResourceNotDefinedError
+from cognite.neat._issues.errors._resources import (
+    ResourceDuplicatedError,
+    ResourceNotDefinedError,
+)
 from cognite.neat._issues.warnings._models import UndefinedClassWarning
 from cognite.neat._issues.warnings._resources import (
     ResourceNotDefinedWarning,
@@ -13,6 +16,7 @@ from cognite.neat._rules._constants import PATTERNS, EntityTypes
 from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
 from cognite.neat._rules.models.entities._multi_value import MultiValueTypeInfo
 from cognite.neat._utils.spreadsheet import SpreadsheetRead
+from cognite.neat._utils.text import humanize_collection
 
 from ._rules import InformationRules
 
@@ -23,13 +27,14 @@ class InformationValidation:
 
     def __init__(self, rules: InformationRules, read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None):
         self.rules = rules
-        self.read_info_by_spreadsheet = read_info_by_spreadsheet
-        self.metadata = rules.metadata
-        self.properties = rules.properties
-        self.classes = rules.classes
+        self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
+        self._metadata = rules.metadata
+        self._properties = rules.properties
+        self._classes = rules.classes
         self.issue_list = IssueList()
 
     def validate(self) -> IssueList:
+        self._duplicated_resources()
         self._namespaces_reassigned()
         self._classes_without_properties()
         self._undefined_classes()
@@ -40,9 +45,51 @@ class InformationValidation:
 
         return self.issue_list
 
+    def _duplicated_resources(self) -> None:
+        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
+        classes_sheet = self._read_info_by_spreadsheet.get("Classes")
+
+        visited = defaultdict(list)
+        for row_no, property_ in enumerate(self._properties):
+            visited[property_._identifier()].append(
+                properties_sheet.adjusted_row_number(row_no) if properties_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            self.issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[1],
+                    "property",
+                    (
+                        "the Properties sheet at row "
+                        f"{humanize_collection(rows)}"
+                        " if data model is read from a spreadsheet."
+                    ),
+                )
+            )
+
+        visited = defaultdict(list)
+        for row_no, class_ in enumerate(self._classes):
+            visited[class_._identifier()].append(
+                classes_sheet.adjusted_row_number(row_no) if classes_sheet else row_no + 1
+            )
+
+        for identifier, rows in visited.items():
+            if len(rows) == 1:
+                continue
+            self.issue_list.append(
+                ResourceDuplicatedError(
+                    identifier[0],
+                    "class",
+                    (f"the Classes sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
+                )
+            )
+
     def _classes_without_properties(self) -> None:
-        defined_classes = {class_.class_ for class_ in self.classes}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        referred_classes = {property_.class_ for property_ in self._properties}
         class_parent_pairs = self._class_parent_pairs()
 
         if classes_without_properties := defined_classes.difference(referred_classes):
@@ -50,7 +97,7 @@ class InformationValidation:
                 # USE CASE: class has no direct properties and no parents with properties
                 # and it is a class in the prefix of data model, as long as it is in the
                 # same prefix, meaning same space
-                if not class_parent_pairs[class_] and class_.prefix == self.metadata.prefix:
+                if not class_parent_pairs[class_] and class_.prefix == self._metadata.prefix:
                     self.issue_list.append(
                         ResourceNotDefinedWarning(
                             resource_type="class",
@@ -60,8 +107,8 @@ class InformationValidation:
                     )
 
     def _undefined_classes(self) -> None:
-        defined_classes = {class_.class_ for class_ in self.classes}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        referred_classes = {property_.class_ for property_ in self._properties}
 
         if undefined_classes := referred_classes.difference(defined_classes):
             for class_ in undefined_classes:
@@ -81,7 +128,7 @@ class InformationValidation:
 
         if undefined_parents := parents.difference(classes):
             for parent in undefined_parents:
-                if parent.prefix != self.metadata.prefix:
+                if parent.prefix != self._metadata.prefix:
                     self.issue_list.append(UndefinedClassWarning(class_id=str(parent)))
                 else:
                     self.issue_list.append(
@@ -94,8 +141,8 @@ class InformationValidation:
 
     def _referenced_classes_exist(self) -> None:
         # needs to be complete for this validation to pass
-        defined_classes = {class_.class_ for class_ in self.classes}
-        classes_with_explicit_properties = {property_.class_ for property_ in self.properties}
+        defined_classes = {class_.class_ for class_ in self._classes}
+        classes_with_explicit_properties = {property_.class_ for property_ in self._properties}
 
         # USE CASE: models are complete
         if missing_classes := classes_with_explicit_properties.difference(defined_classes):
@@ -110,7 +157,7 @@ class InformationValidation:
 
     def _referenced_value_types_exist(self) -> None:
         # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
-        defined_classes = {class_.class_ for class_ in self.classes} | {UnknownEntity()}
+        defined_classes = {class_.class_ for class_ in self._classes} | {UnknownEntity()}
         referred_object_types = {
             property_.value_type
             for property_ in self.rules.properties
@@ -131,7 +178,7 @@ class InformationValidation:
     def _regex_compliance_with_dms(self) -> None:
         """Check regex compliance with DMS of properties, classes and value types."""
 
-        for prop_ in self.properties:
+        for prop_ in self._properties:
             if not PATTERNS.dms_property_id_compliance.match(prop_.property_):
                 self.issue_list.append(
                     ResourceRegexViolationWarning(
@@ -181,7 +228,7 @@ class InformationValidation:
                             )
                         )
 
-        for class_ in self.classes:
+        for class_ in self._classes:
             if not PATTERNS.view_id_compliance.match(class_.class_.suffix):
                 self.issue_list.append(
                     ResourceRegexViolationWarning(

cognite/neat/_rules/transformers/_converters.py

@@ -60,7 +60,7 @@ from cognite.neat._rules.models.entities import (
 )
 from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
 from cognite.neat._utils.rdf_ import get_inheritance_path
-from cognite.neat._utils.text import NamingStandardization, to_camel_case
+from cognite.neat._utils.text import NamingStandardization, title, to_camel_case, to_words
 
 from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
 from ._verification import VerifyDMSRules
@@ -561,8 +561,9 @@ _T_Entity = TypeVar("_T_Entity", bound=ClassEntity | ViewEntity)
 
 
 class SetIDDMSModel(VerifiedRulesTransformer[DMSRules, DMSRules]):
-    def __init__(self, new_id: DataModelId | tuple[str, str, str]):
+    def __init__(self, new_id: DataModelId | tuple[str, str, str], name: str | None = None):
         self.new_id = DataModelId.load(new_id)
+        self.name = name
 
     @property
     def description(self) -> str:
@@ -575,10 +576,14 @@ class SetIDDMSModel(VerifiedRulesTransformer[DMSRules, DMSRules]):
         dump["metadata"]["space"] = self.new_id.space
         dump["metadata"]["external_id"] = self.new_id.external_id
         dump["metadata"]["version"] = self.new_id.version
+        dump["metadata"]["name"] = self.name or self._generate_name()
         # Serialize and deserialize to set the new space and external_id
         # as the default values for the new model.
         return DMSRules.model_validate(DMSInputRules.load(dump).dump())
 
+    def _generate_name(self) -> str:
+        return title(to_words(self.new_id.external_id))
+
 
 class ToExtensionModel(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
     type_: ClassVar[str]

cognite/neat/_session/_base.py

@@ -25,6 +25,7 @@ from cognite.neat._utils.auxiliary import local_import
 from ._collector import _COLLECTOR, Collector
 from ._create import CreateAPI
 from ._drop import DropAPI
+from ._explore import ExploreAPI
 from ._fix import FixAPI
 from ._inspect import InspectAPI
 from ._mapping import MappingAPI
@@ -104,6 +105,7 @@ class NeatSession:
         self.drop = DropAPI(self._state)
         self.subset = SubsetAPI(self._state)
         self.create = CreateAPI(self._state)
+        self._explore = ExploreAPI(self._state)
         self.opt = OptAPI()
         self.opt._display()
         if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):

cognite/neat/_session/_explore.py

@@ -0,0 +1,39 @@
+from typing import cast
+
+import pandas as pd
+from rdflib import URIRef
+
+from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._utils.text import humanize_collection
+
+from ._state import SessionState
+from .exceptions import NeatSessionError, session_class_wrapper
+
+
+@session_class_wrapper
+class ExploreAPI:
+    """
+    Explore the instances in the session.
+    """
+
+    def __init__(self, state: SessionState):
+        self._state = state
+
+    def types(self) -> pd.DataFrame:
+        """List all the types of instances in the session."""
+        return pd.DataFrame(self._state.instances.store.queries.types_with_instance_and_property_count())
+
+    def properties(self) -> pd.DataFrame:
+        """List all the properties of a type of instances in the session."""
+        return pd.DataFrame(self._state.instances.store.queries.properties_with_count())
+
+    def instance_with_properties(self, type: str) -> dict[str, set[str]]:
+        """List all the instances of a type with their properties."""
+        available_types = self._state.instances.store.queries.list_types(remove_namespace=False)
+        uri_by_type = {remove_namespace_from_uri(t[0]): t[0] for t in available_types}
+        if type not in uri_by_type:
+            raise NeatSessionError(
+                f"Type {type} not found. Available types are: {humanize_collection(uri_by_type.keys())}"
+            )
+        type_uri = cast(URIRef, uri_by_type[type])
+        return self._state.instances.store.queries.instances_with_properties(type_uri, remove_namespace=True)