cognite-neat 0.76.1__py3-none-any.whl → 0.76.2__py3-none-any.whl

cognite/neat/rules/models/dms/_serializer.py

@@ -0,0 +1,126 @@
+from typing import Any, ClassVar, cast
+
+from pydantic_core.core_schema import SerializationInfo
+
+from cognite.neat.rules.models import DMSRules
+from cognite.neat.rules.models.dms import DMSContainer, DMSProperty, DMSView
+
+
+class _DMSRulesSerializer:
+    # These are the fields that need to be cleaned from the default space and version
+    PROPERTIES_FIELDS: ClassVar[list[str]] = ["class_", "view", "value_type", "container"]
+    VIEWS_FIELDS: ClassVar[list[str]] = ["class_", "view", "implements"]
+    CONTAINERS_FIELDS: ClassVar[list[str]] = ["class_", "container"]
+
+    def __init__(self, info: SerializationInfo, default_space: str, default_version: str) -> None:
+        self.default_space = f"{default_space}:"
+        self.default_version = f"version={default_version}"
+        self.default_version_wrapped = f"({self.default_version})"
+
+        self.properties_fields = self.PROPERTIES_FIELDS
+        self.views_fields = self.VIEWS_FIELDS
+        self.containers_fields = self.CONTAINERS_FIELDS
+        self.prop_name = "properties"
+        self.view_name = "views"
+        self.container_name = "containers"
+        self.metadata_name = "metadata"
+        self.prop_view = "view"
+        self.prop_view_property = "view_property"
+        self.prop_value_type = "value_type"
+        self.view_view = "view"
+        self.view_implements = "implements"
+        self.container_container = "container"
+        self.container_constraint = "constraint"
+
+        if info.by_alias:
+            self.properties_fields = [
+                DMSProperty.model_fields[field].alias or field for field in self.properties_fields
+            ]
+            self.views_fields = [DMSView.model_fields[field].alias or field for field in self.views_fields]
+            self.container_fields = [
+                DMSContainer.model_fields[field].alias or field for field in self.containers_fields
+            ]
+            self.prop_view = DMSProperty.model_fields[self.prop_view].alias or self.prop_view
+            self.prop_view_property = DMSProperty.model_fields[self.prop_view_property].alias or self.prop_view_property
+            self.prop_value_type = DMSProperty.model_fields[self.prop_value_type].alias or self.prop_value_type
+            self.view_view = DMSView.model_fields[self.view_view].alias or self.view_view
+            self.view_implements = DMSView.model_fields[self.view_implements].alias or self.view_implements
+            self.container_container = (
+                DMSContainer.model_fields[self.container_container].alias or self.container_container
+            )
+            self.container_constraint = (
+                DMSContainer.model_fields[self.container_constraint].alias or self.container_constraint
+            )
+            self.prop_name = DMSRules.model_fields[self.prop_name].alias or self.prop_name
+            self.view_name = DMSRules.model_fields[self.view_name].alias or self.view_name
+            self.container_name = DMSRules.model_fields[self.container_name].alias or self.container_name
+            self.metadata_name = DMSRules.model_fields[self.metadata_name].alias or self.metadata_name
+
+        if isinstance(info.exclude, dict):
+            # Just for happy mypy
+            exclude = cast(dict, info.exclude)
+            self.exclude_properties = exclude.get("properties", {}).get("__all__", set())
+            self.exclude_views = exclude.get("views", {}).get("__all__", set()) or set()
+            self.exclude_containers = exclude.get("containers", {}).get("__all__", set()) or set()
+            self.metadata_exclude = exclude.get("metadata", set()) or set()
+            self.exclude_top = {k for k, v in exclude.items() if not v}
+        else:
+            self.exclude_top = set(info.exclude or {})
+            self.exclude_properties = set()
+            self.exclude_views = set()
+            self.exclude_containers = set()
+            self.metadata_exclude = set()
+
+    def clean(self, dumped: dict[str, Any]) -> dict[str, Any]:
+        # Sorting to get a deterministic order
+        dumped[self.prop_name] = sorted(
+            dumped[self.prop_name]["data"], key=lambda p: (p[self.prop_view], p[self.prop_view_property])
+        )
+        dumped[self.view_name] = sorted(dumped[self.view_name]["data"], key=lambda v: v[self.view_view])
+        if self.container_name in dumped:
+            dumped[self.container_name] = sorted(
+                dumped[self.container_name]["data"], key=lambda c: c[self.container_container]
+            )
+
+        for prop in dumped[self.prop_name]:
+            for field_name in self.properties_fields:
+                if value := prop.get(field_name):
+                    prop[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
+            # Value type can have a property as well
+            prop[self.prop_value_type] = prop[self.prop_value_type].replace(self.default_version, "")
+            if self.exclude_properties:
+                for field in self.exclude_properties:
+                    prop.pop(field, None)
+
+        for view in dumped[self.view_name]:
+            for field_name in self.views_fields:
+                if value := view.get(field_name):
+                    view[field_name] = value.removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
+            if value := view.get(self.view_implements):
+                view[self.view_implements] = ",".join(
+                    parent.strip().removeprefix(self.default_space).removesuffix(self.default_version_wrapped)
+                    for parent in value.split(",")
+                )
+            if self.exclude_views:
+                for field in self.exclude_views:
+                    view.pop(field, None)
+
+        for container in dumped[self.container_name]:
+            for field_name in self.containers_fields:
+                if value := container.get(field_name):
+                    container[field_name] = value.removeprefix(self.default_space)
+
+            if value := container.get(self.container_constraint):
+                container[self.container_constraint] = ",".join(
+                    constraint.strip().removeprefix(self.default_space) for constraint in value.split(",")
+                )
+            if self.exclude_containers:
+                for field in self.exclude_containers:
+                    container.pop(field, None)
+
+        if self.metadata_exclude:
+            for field in self.metadata_exclude:
+                dumped[self.metadata_name].pop(field, None)
+        for field in self.exclude_top:
+            dumped.pop(field, None)
+        return dumped

cognite/neat/rules/models/dms/_validation.py

@@ -0,0 +1,255 @@
+from collections import defaultdict
+from typing import Any
+
+from cognite.neat.rules import issues
+from cognite.neat.rules.issues import IssueList
+from cognite.neat.rules.models._base import ExtensionCategory, SchemaCompleteness
+from cognite.neat.rules.models.data_types import DataType
+from cognite.neat.rules.models.entities import ContainerEntity
+
+from ._rules import DMSProperty, DMSRules
+
+
+class DMSPostValidation:
+    """This class does all the validation of the DMS rules that have dependencies between
+    components."""
+
+    def __init__(self, rules: DMSRules):
+        self.rules = rules
+        self.metadata = rules.metadata
+        self.properties = rules.properties
+        self.containers = rules.containers
+        self.views = rules.views
+        self.issue_list = IssueList()
+
+    def validate(self) -> IssueList:
+        self._consistent_container_properties()
+        self._referenced_views_and_containers_are_existing()
+        self._validate_extension()
+        self._validate_schema()
+        return self.issue_list
+
+    def _consistent_container_properties(self) -> None:
+        container_properties_by_id: dict[tuple[ContainerEntity, str], list[tuple[int, DMSProperty]]] = defaultdict(list)
+        for prop_no, prop in enumerate(self.properties):
+            if prop.container and prop.container_property:
+                container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
+
+        errors: list[issues.spreadsheet.InconsistentContainerDefinitionError] = []
+        for (container, prop_name), properties in container_properties_by_id.items():
+            if len(properties) == 1:
+                continue
+            container_id = container.as_id()
+            row_numbers = {prop_no for prop_no, _ in properties}
+            value_types = {prop.value_type for _, prop in properties if prop.value_type}
+            if len(value_types) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiValueTypeError(
+                        container_id,
+                        prop_name,
+                        row_numbers,
+                        {v.dms._type if isinstance(v, DataType) else str(v) for v in value_types},
+                    )
+                )
+            list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
+            if len(list_definitions) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiValueIsListError(container_id, prop_name, row_numbers, list_definitions)
+                )
+            nullable_definitions = {prop.nullable for _, prop in properties if prop.nullable is not None}
+            if len(nullable_definitions) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiNullableError(container_id, prop_name, row_numbers, nullable_definitions)
+                )
+            default_definitions = {prop.default for _, prop in properties if prop.default is not None}
+            if len(default_definitions) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiDefaultError(
+                        container_id, prop_name, row_numbers, list(default_definitions)
+                    )
+                )
+            index_definitions = {",".join(prop.index) for _, prop in properties if prop.index is not None}
+            if len(index_definitions) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiIndexError(container_id, prop_name, row_numbers, index_definitions)
+                )
+            constraint_definitions = {
+                ",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
+            }
+            if len(constraint_definitions) > 1:
+                errors.append(
+                    issues.spreadsheet.MultiUniqueConstraintError(
+                        container_id, prop_name, row_numbers, constraint_definitions
+                    )
+                )
+
+            # This sets the container definition for all the properties where it is not defined.
+            # This allows the user to define the container only once.
+            value_type = next(iter(value_types))
+            list_definition = next(iter(list_definitions)) if list_definitions else None
+            nullable_definition = next(iter(nullable_definitions)) if nullable_definitions else None
+            default_definition = next(iter(default_definitions)) if default_definitions else None
+            index_definition = next(iter(index_definitions)).split(",") if index_definitions else None
+            constraint_definition = next(iter(constraint_definitions)).split(",") if constraint_definitions else None
+            for _, prop in properties:
+                prop.value_type = value_type
+                prop.is_list = prop.is_list or list_definition
+                prop.nullable = prop.nullable or nullable_definition
+                prop.default = prop.default or default_definition
+                prop.index = prop.index or index_definition
+                prop.constraint = prop.constraint or constraint_definition
+        self.issue_list.extend(errors)
+
+    def _referenced_views_and_containers_are_existing(self) -> None:
+        # There two checks are done in the same method to raise all the errors at once.
+        defined_views = {view.view.as_id() for view in self.views}
+
+        errors: list[issues.NeatValidationError] = []
+        for prop_no, prop in enumerate(self.properties):
+            if prop.view and (view_id := prop.view.as_id()) not in defined_views:
+                errors.append(
+                    issues.spreadsheet.NonExistingViewError(
+                        column="View",
+                        row=prop_no,
+                        type="value_error.missing",
+                        view_id=view_id,
+                        msg="",
+                        input=None,
+                        url=None,
+                    )
+                )
+        if self.metadata.schema_ is SchemaCompleteness.complete:
+            defined_containers = {container.container.as_id() for container in self.containers or []}
+            for prop_no, prop in enumerate(self.properties):
+                if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
+                    errors.append(
+                        issues.spreadsheet.NonExistingContainerError(
+                            column="Container",
+                            row=prop_no,
+                            type="value_error.missing",
+                            container_id=container_id,
+                            msg="",
+                            input=None,
+                            url=None,
+                        )
+                    )
+            for _container_no, container in enumerate(self.containers or []):
+                for constraint_no, constraint in enumerate(container.constraint or []):
+                    if constraint.as_id() not in defined_containers:
+                        errors.append(
+                            issues.spreadsheet.NonExistingContainerError(
+                                column="Constraint",
+                                row=constraint_no,
+                                type="value_error.missing",
+                                container_id=constraint.as_id(),
+                                msg="",
+                                input=None,
+                                url=None,
+                            )
+                        )
+        self.issue_list.extend(errors)
+
+    def _validate_extension(self) -> None:
+        if self.metadata.schema_ is not SchemaCompleteness.extended:
+            return None
+        if not self.rules.reference:
+            raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
+        is_solution = self.metadata.space != self.rules.reference.metadata.space
+        if is_solution:
+            return None
+        if self.metadata.extension is ExtensionCategory.rebuild:
+            # Everything is allowed
+            return None
+        # Is an extension of an existing model.
+        user_schema = self.rules.as_schema(include_ref=False)
+        ref_schema = self.rules.reference.as_schema()
+        new_containers = {container.as_id(): container for container in user_schema.containers}
+        existing_containers = {container.as_id(): container for container in ref_schema.containers}
+
+        for container_id, container in new_containers.items():
+            existing_container = existing_containers.get(container_id)
+            if not existing_container or existing_container == container:
+                # No problem
+                continue
+            new_dumped = container.dump()
+            existing_dumped = existing_container.dump()
+            changed_attributes, changed_properties = self._changed_attributes_and_properties(
+                new_dumped, existing_dumped
+            )
+            self.issue_list.append(
+                issues.dms.ChangingContainerError(
+                    container_id=container_id,
+                    changed_properties=changed_properties or None,
+                    changed_attributes=changed_attributes or None,
+                )
+            )
+
+        if self.metadata.extension is ExtensionCategory.reshape and self.issue_list:
+            return None
+        elif self.metadata.extension is ExtensionCategory.reshape:
+            # Reshape allows changes to views
+            return None
+
+        new_views = {view.as_id(): view for view in user_schema.views}
+        existing_views = {view.as_id(): view for view in ref_schema.views}
+        for view_id, view in new_views.items():
+            existing_view = existing_views.get(view_id)
+            if not existing_view or existing_view == view:
+                # No problem
+                continue
+            changed_attributes, changed_properties = self._changed_attributes_and_properties(
+                view.dump(), existing_view.dump()
+            )
+            self.issue_list.append(
+                issues.dms.ChangingViewError(
+                    view_id=view_id,
+                    changed_properties=changed_properties or None,
+                    changed_attributes=changed_attributes or None,
+                )
+            )
+
+    @staticmethod
+    def _changed_attributes_and_properties(
+        new_dumped: dict[str, Any], existing_dumped: dict[str, Any]
+    ) -> tuple[list[str], list[str]]:
+        """Helper method to find the changed attributes and properties between two containers or views."""
+        new_attributes = {key: value for key, value in new_dumped.items() if key != "properties"}
+        existing_attributes = {key: value for key, value in existing_dumped.items() if key != "properties"}
+        changed_attributes = [key for key in new_attributes if new_attributes[key] != existing_attributes.get(key)]
+        new_properties = new_dumped.get("properties", {})
+        existing_properties = existing_dumped.get("properties", {})
+        changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
+        return changed_attributes, changed_properties
+
+    def _validate_schema(self) -> None:
+        if self.metadata.schema_ is SchemaCompleteness.partial:
+            return None
+        elif self.metadata.schema_ is SchemaCompleteness.complete:
+            rules: DMSRules = self.rules
+        elif self.metadata.schema_ is SchemaCompleteness.extended:
+            if not self.rules.reference:
+                raise ValueError(
+                    "The schema is set to 'extended', but no reference rules are provided to validate against"
+                )
+            # This is an extension of the reference rules, we need to merge the two
+            rules = self.rules.model_copy(deep=True)
+            rules.properties.extend(self.rules.reference.properties.data)
+            existing_views = {view.view.as_id() for view in rules.views}
+            rules.views.extend([view for view in self.rules.reference.views if view.view.as_id() not in existing_views])
+            if rules.containers and self.rules.reference.containers:
+                existing_containers = {container.container.as_id() for container in rules.containers.data}
+                rules.containers.extend(
+                    [
+                        container
+                        for container in self.rules.reference.containers
+                        if container.container.as_id() not in existing_containers
+                    ]
+                )
+            elif not rules.containers and self.rules.reference.containers:
+                rules.containers = self.rules.reference.containers
+        else:
+            raise ValueError("Unknown schema completeness")
+
+        schema = rules.as_schema()
+        errors = schema.validate()
+        self.issue_list.extend(errors)

cognite/neat/rules/models/information/__init__.py

@@ -0,0 +1,3 @@
+from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
+
+__all__ = ["InformationRules", "InformationMetadata", "InformationClass", "InformationProperty"]

cognite/neat/rules/models/information/_converter.py

@@ -0,0 +1,193 @@
+import re
+from collections import defaultdict
+from datetime import datetime
+from typing import Literal
+
+from cognite.neat.rules.models._base import (
+    SheetList,
+)
+from cognite.neat.rules.models.data_types import DataType
+from cognite.neat.rules.models.dms._rules import DMSProperty, DMSRules, DMSView
+from cognite.neat.rules.models.domain import DomainRules
+from cognite.neat.rules.models.entities import (
+    ClassEntity,
+    ContainerEntity,
+    DMSUnknownEntity,
+    ReferenceEntity,
+    UnknownEntity,
+    ViewEntity,
+    ViewPropertyEntity,
+)
+
+from ._rules import InformationClass, InformationMetadata, InformationProperty, InformationRules
+
+
+class _InformationRulesConverter:
+    def __init__(self, information: InformationRules):
+        self.information = information
+
+    def as_domain_rules(self) -> DomainRules:
+        raise NotImplementedError("DomainRules not implemented yet")
+
+    def as_dms_architect_rules(self, created: datetime | None = None, updated: datetime | None = None) -> "DMSRules":
+        from cognite.neat.rules.models.dms._rules import (
+            DMSContainer,
+            DMSMetadata,
+            DMSProperty,
+            DMSRules,
+        )
+
+        info_metadata = self.information.metadata
+        default_version = info_metadata.version
+        default_space = self._to_space(info_metadata.prefix)
+        space = self._to_space(info_metadata.prefix)
+
+        metadata = DMSMetadata(
+            schema_=info_metadata.schema_,
+            space=space,
+            version=info_metadata.version,
+            external_id=info_metadata.name.replace(" ", "_").lower(),
+            creator=info_metadata.creator,
+            name=info_metadata.name,
+            created=created or datetime.now(),
+            updated=updated or datetime.now(),
+        )
+
+        properties_by_class: dict[str, list[DMSProperty]] = defaultdict(list)
+        for prop in self.information.properties:
+            properties_by_class[prop.class_.versioned_id].append(
+                self._as_dms_property(prop, default_space, default_version)
+            )
+
+        views: list[DMSView] = [
+            DMSView(
+                class_=cls_.class_,
+                name=cls_.name,
+                view=cls_.class_.as_view_entity(default_space, default_version),
+                description=cls_.description,
+                reference=cls_.reference,
+                implements=self._get_view_implements(cls_, info_metadata),
+            )
+            for cls_ in self.information.classes
+        ]
+
+        classes_without_properties: set[str] = set()
+        for class_ in self.information.classes:
+            properties: list[DMSProperty] = properties_by_class.get(class_.class_.versioned_id, [])
+            if not properties or all(
+                isinstance(prop.value_type, ViewPropertyEntity) and prop.connection != "direct" for prop in properties
+            ):
+                classes_without_properties.add(class_.class_.versioned_id)
+
+        containers: list[DMSContainer] = []
+        for class_ in self.information.classes:
+            if class_.class_.versioned_id in classes_without_properties:
+                continue
+            containers.append(
+                DMSContainer(
+                    class_=class_.class_,
+                    name=class_.name,
+                    container=class_.class_.as_container_entity(default_space),
+                    description=class_.description,
+                    constraint=[
+                        parent.as_container_entity(default_space)
+                        for parent in class_.parent or []
+                        if parent.versioned_id not in classes_without_properties
+                    ]
+                    or None,
+                )
+            )
+
+        return DMSRules(
+            metadata=metadata,
+            properties=SheetList[DMSProperty](
+                data=[prop for prop_set in properties_by_class.values() for prop in prop_set]
+            ),
+            views=SheetList[DMSView](data=views),
+            containers=SheetList[DMSContainer](data=containers),
+            last=self.information.last.as_dms_architect_rules() if self.information.last else None,
+            reference=self.information.reference.as_dms_architect_rules() if self.information.reference else None,
+        )
+
+    @classmethod
+    def _as_dms_property(cls, prop: InformationProperty, default_space: str, default_version: str) -> "DMSProperty":
+        """This creates the first"""
+
+        from cognite.neat.rules.models.dms._rules import DMSProperty
+
+        # returns property type, which can be ObjectProperty or DatatypeProperty
+        value_type: DataType | ViewEntity | ViewPropertyEntity | DMSUnknownEntity
+        if isinstance(prop.value_type, DataType):
+            value_type = prop.value_type
+        elif isinstance(prop.value_type, UnknownEntity):
+            value_type = DMSUnknownEntity()
+        elif isinstance(prop.value_type, ClassEntity):
+            value_type = prop.value_type.as_view_entity(default_space, default_version)
+        else:
+            raise ValueError(f"Unsupported value type: {prop.value_type.type_}")
+
+        relation: Literal["direct", "edge", "reverse"] | None = None
+        if isinstance(value_type, ViewEntity | ViewPropertyEntity):
+            relation = "edge" if prop.is_list else "direct"
+
+        container: ContainerEntity | None = None
+        container_property: str | None = None
+        is_list: bool | None = prop.is_list
+        nullable: bool | None = not prop.is_mandatory
+        if relation == "edge":
+            nullable = None
+        elif relation == "direct":
+            nullable = True
+            container, container_property = cls._get_container(prop, default_space)
+        else:
+            container, container_property = cls._get_container(prop, default_space)
+
+        return DMSProperty(
+            class_=prop.class_,
+            name=prop.name,
+            property_=prop.property_,
+            value_type=value_type,
+            nullable=nullable,
+            is_list=is_list,
+            connection=relation,
+            default=prop.default,
+            reference=prop.reference,
+            container=container,
+            container_property=container_property,
+            view=prop.class_.as_view_entity(default_space, default_version),
+            view_property=prop.property_,
+        )
+
+    @classmethod
+    def _to_space(cls, prefix: str) -> str:
+        """Ensures that the prefix comply with the CDF space regex"""
+        prefix = re.sub(r"[^a-zA-Z0-9_-]", "_", prefix)
+        if prefix[0].isdigit() or prefix[0] == "_":
+            prefix = f"a{prefix}"
+        prefix = prefix[:43]
+        if prefix[-1] == "_":
+            prefix = f"{prefix[:-1]}1"
+        return prefix
+
+    @classmethod
+    def _get_container(cls, prop: InformationProperty, default_space: str) -> tuple[ContainerEntity, str]:
+        if isinstance(prop.reference, ReferenceEntity):
+            return (
+                prop.reference.as_container_entity(default_space),
+                prop.reference.property_ or prop.property_,
+            )
+        else:
+            return prop.class_.as_container_entity(default_space), prop.property_
+
+    @classmethod
+    def _get_view_implements(cls, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
+        if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:
+            # We use the reference for implements if it is in a different namespace
+            implements = [
+                cls_.reference.as_view_entity(metadata.prefix, metadata.version),
+            ]
+        else:
+            implements = []
+
+        implements.extend([parent.as_view_entity(metadata.prefix, metadata.version) for parent in cls_.parent or []])
+        return implements