cognite-neat 0.77.6__py3-none-any.whl → 0.77.8__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.77.6"
+__version__ = "0.77.8"

cognite/neat/rules/issues/importing.py

@@ -16,6 +16,7 @@ __all__ = [
     "MultipleDataModelsWarning",
     "UnknownPropertyTypeWarning",
     "FailedToInferValueTypeWarning",
+    "MoreThanOneNonAlphanumericCharacterWarning",
     "UnknownContainerConstraintWarning",
     "NoDataModelError",
     "ModelImportError",
@@ -275,6 +276,22 @@ class FailedImportWarning(ModelImportWarning):
         return {"identifier": list(self.identifier)}
 
 
+@dataclass(frozen=True)
+class MoreThanOneNonAlphanumericCharacterWarning(ModelImportWarning):
+    description = """This warning is raised when doing regex validation of strings which either represent class ids,
+    property ids, prefix, data model name, that contain more than one non-alphanumeric character,
+    such as for example '_' or '-'."""
+
+    field_name: str
+    value: str
+
+    def message(self) -> str:
+        return f"Field {self.field_name} with value {self.value} contains more than one non-alphanumeric character!"
+
+    def dump(self) -> dict[str, str]:
+        return {"field_name": self.field_name, "value": self.value}
+
+
 @dataclass(frozen=True)
 class ModelImportError(NeatValidationError, ABC):
     description = "An error was raised during importing."

cognite/neat/rules/models/_types/_field.py

@@ -1,4 +1,5 @@
 import re
+import warnings
 from collections.abc import Callable
 from typing import Annotated, Any, cast
 
@@ -17,6 +18,7 @@ from pydantic.functional_serializers import PlainSerializer
 from pydantic_core import PydanticCustomError
 
 from cognite.neat.rules import exceptions
+from cognite.neat.rules.issues.importing import MoreThanOneNonAlphanumericCharacterWarning
 
 from ._base import (
     MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX,
@@ -50,7 +52,6 @@ StrOrListType = Annotated[
     ),
 ]
 
-
 StrListType = Annotated[
     list[str],
     BeforeValidator(lambda value: [entry.strip() for entry in value.split(",")] if isinstance(value, str) else value),
@@ -96,19 +97,12 @@ VersionType = Annotated[
 ]
 
 
-PropertyType = Annotated[
-    str,
-    AfterValidator(
-        lambda value: (
-            _raise(exceptions.MoreThanOneNonAlphanumericCharacter("property", value).to_pydantic_custom_error())
-            if re.search(MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX, value)
-            else (
-                value
-                if re.match(PROPERTY_ID_COMPLIANCE_REGEX, value)
-                else _raise(
-                    exceptions.PropertyIDRegexViolation(value, PROPERTY_ID_COMPLIANCE_REGEX).to_pydantic_custom_error()
-                )
-            )
-        )
-    ),
-]
+def _property_validation(value: str) -> str:
+    if not re.match(PROPERTY_ID_COMPLIANCE_REGEX, value):
+        _raise(exceptions.PropertyIDRegexViolation(value, PROPERTY_ID_COMPLIANCE_REGEX).to_pydantic_custom_error())
+    if re.search(MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX, value):
+        warnings.warn(MoreThanOneNonAlphanumericCharacterWarning("property", value), stacklevel=2)
+    return value
+
+
+PropertyType = Annotated[str, AfterValidator(_property_validation)]

cognite/neat/rules/models/dms/_validation.py

@@ -57,7 +57,10 @@ class DMSPostValidation:
             container_id = container.as_id()
             row_numbers = {prop_no for prop_no, _ in properties}
             value_types = {prop.value_type for _, prop in properties if prop.value_type}
-            if len(value_types) > 1:
+            # The container type 'direct' is an exception. On a container the type direct can point to any
+            # node. The value type is typically set on the view.
+            is_all_direct = all(prop.connection == "direct" for _, prop in properties)
+            if len(value_types) > 1 and not is_all_direct:
                 errors.append(
                     issues.spreadsheet.MultiValueTypeError(
                         container_id,

cognite/neat/rules/models/information/_converter.py

@@ -1,5 +1,6 @@
 import re
-from collections import defaultdict
+from collections import Counter, defaultdict
+from collections.abc import Collection
 from typing import TYPE_CHECKING, Literal
 
 from cognite.neat.rules.models._base import (
@@ -40,7 +41,6 @@ class _InformationRulesConverter:
             self.last_classes = {class_.class_: class_ for class_ in self.rules.last.classes}
         else:
             self.last_classes = {}
-        self._created_classes_from: dict[ClassEntity, ClassEntity] = {}
 
     def as_domain_rules(self) -> DomainRules:
         raise NotImplementedError("DomainRules not implemented yet")
@@ -58,11 +58,13 @@ class _InformationRulesConverter:
         default_space = self._to_space(info_metadata.prefix)
         metadata = self._convert_metadata_to_dms(info_metadata)
 
-        properties_by_class: dict[str, list[DMSProperty]] = defaultdict(list)
+        properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
+        referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
         for prop in self.rules.properties:
-            properties_by_class[prop.class_.versioned_id].append(
-                self._as_dms_property(prop, default_space, default_version)
-            )
+            dms_property = self._as_dms_property(prop, default_space, default_version)
+            properties_by_class[prop.class_].append(dms_property)
+            if dms_property.container:
+                referenced_containers[dms_property.container][prop.class_] += 1
 
         views: list[DMSView] = [
             DMSView(
@@ -76,41 +78,37 @@ class _InformationRulesConverter:
             for cls_ in self.rules.classes
         ]
 
-        classes_without_properties: set[str] = set()
-        for class_ in self.rules.classes:
-            properties: list[DMSProperty] = properties_by_class.get(class_.class_.versioned_id, [])
-            if not properties or all(
-                isinstance(prop.value_type, ViewPropertyEntity) and prop.connection != "direct" for prop in properties
-            ):
-                classes_without_properties.add(class_.class_.versioned_id)
+        last_dms_rules = self.rules.last.as_dms_architect_rules() if self.rules.last else None
+        ref_dms_rules = self.rules.reference.as_dms_architect_rules() if self.rules.reference else None
 
         containers: list[DMSContainer] = []
-        classes = list(self.rules.classes)
-        for new_class_entity, created_from in self._created_classes_from.items():
-            # We create new classes in case metadata is set to addition or reshape
-            # and the class was present in the last schema to avoid creating
-            # a changed container and instead create a new one.
-            created_class = self.last_classes[created_from].copy(deep=True)
-            created_class.class_ = new_class_entity
-            classes.append(created_class)
-
-        for class_ in classes:
-            if class_.class_.versioned_id in classes_without_properties:
+        class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
+        if self.rules.last:
+            for cls_ in self.rules.last.classes:
+                if cls_.class_ not in class_by_entity:
+                    class_by_entity[cls_.class_] = cls_
+
+        existing_containers: set[ContainerEntity] = set()
+        for rule_set in [last_dms_rules, ref_dms_rules]:
+            if rule_set:
+                existing_containers.update({c.container for c in rule_set.containers or []})
+
+        for container_entity, class_entities in referenced_containers.items():
+            if container_entity in existing_containers:
                 continue
-            containers.append(
-                DMSContainer(
-                    class_=class_.class_,
-                    name=class_.name,
-                    container=class_.class_.as_container_entity(default_space),
-                    description=class_.description,
-                    constraint=[
-                        parent.as_container_entity(default_space)
-                        for parent in class_.parent or []
-                        if parent.versioned_id not in classes_without_properties
-                    ]
-                    or None,
-                )
+            constrains = self._create_container_constraint(
+                class_entities, default_space, class_by_entity, referenced_containers
+            )
+            most_used_class_entity = class_entities.most_common(1)[0][0]
+            class_ = class_by_entity[most_used_class_entity]
+            container = DMSContainer(
+                class_=class_.class_,
+                container=container_entity,
+                name=class_.name,
+                description=class_.description,
+                constraint=constrains or None,
             )
+            containers.append(container)
 
         return DMSRules(
             metadata=metadata,
@@ -119,10 +117,26 @@ class _InformationRulesConverter:
             ),
             views=SheetList[DMSView](data=views),
             containers=SheetList[DMSContainer](data=containers),
-            last=self.rules.last.as_dms_architect_rules() if self.rules.last else None,
-            reference=self.rules.reference.as_dms_architect_rules() if self.rules.reference else None,
+            last=last_dms_rules,
+            reference=ref_dms_rules,
         )
 
+    @staticmethod
+    def _create_container_constraint(
+        class_entities: Counter[ClassEntity],
+        default_space: str,
+        class_by_entity: dict[ClassEntity, InformationClass],
+        referenced_containers: Collection[ContainerEntity],
+    ) -> list[ContainerEntity]:
+        constrains: list[ContainerEntity] = []
+        for entity in class_entities:
+            class_ = class_by_entity[entity]
+            for parent in class_.parent or []:
+                parent_entity = parent.as_container_entity(default_space)
+                if parent_entity in referenced_containers:
+                    constrains.append(parent_entity)
+        return constrains
+
     @classmethod
     def _convert_metadata_to_dms(cls, metadata: InformationMetadata) -> "DMSMetadata":
         from cognite.neat.rules.models.dms._rules import (
@@ -211,25 +225,35 @@ class _InformationRulesConverter:
         elif (self.is_addition or self.is_reshape) and prop.class_ in self.last_classes:
             # We need to create a new container for the property, as we cannot change
             # the existing container in the last schema
-            class_entity = prop.class_.copy()
-            class_entity.suffix = self._bump_suffix(class_entity.suffix)
-            if class_entity not in self._created_classes_from:
-                self._created_classes_from[class_entity] = prop.class_
-            return class_entity.as_container_entity(default_space), prop.property_
+            container_entity = prop.class_.as_container_entity(default_space)
+            container_entity.suffix = self._bump_suffix(container_entity.suffix)
+            return container_entity, prop.property_
         else:
             return prop.class_.as_container_entity(default_space), prop.property_
 
-    @classmethod
-    def _get_view_implements(cls, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
+    def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
         if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:
             # We use the reference for implements if it is in a different namespace
-            implements = [
-                cls_.reference.as_view_entity(metadata.prefix, metadata.version),
-            ]
+            if self.rules.reference and cls_.reference.prefix == self.rules.reference.metadata.prefix:
+                implements = [
+                    cls_.reference.as_view_entity(
+                        self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
+                    )
+                ]
+            else:
+                implements = [
+                    cls_.reference.as_view_entity(metadata.prefix, metadata.version),
+                ]
         else:
             implements = []
-
-        implements.extend([parent.as_view_entity(metadata.prefix, metadata.version) for parent in cls_.parent or []])
+        for parent in cls_.parent or []:
+            if self.rules.reference and parent.prefix == self.rules.reference.metadata.prefix:
+                view_entity = parent.as_view_entity(
+                    self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
+                )
+            else:
+                view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
+            implements.append(view_entity)
         return implements
 
     @staticmethod

cognite/neat/workflows/steps/lib/current/rules_exporter.py

@@ -281,7 +281,6 @@ class RulesToExcel(Step):
             value="",
             label="If you chose Dump Format 'reference', the provided ID will be use in the new medata sheet. "
             "Expected format 'sp_space:my_external_id'.",
-            options=list(exporters.ExcelExporter.dump_options),
         ),
         Configurable(
             name="File path",

{cognite_neat-0.77.6.dist-info → cognite_neat-0.77.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.77.6
+Version: 0.77.8
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.77.6.dist-info → cognite_neat-0.77.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=gRPwu1nLGE_QfxUJtmhbOlmhiHQAZRF63Q3wmAlYElY,23
+cognite/neat/_version.py,sha256=mXpwNIS8gqWNHN489kDHmqkjGuUNEJ_NwDA07_xdNQk,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -190,7 +190,7 @@ cognite/neat/rules/issues/base.py,sha256=i2aTC-wq3UVW2bj_7wKeuhYxCpMD06Bd9-m00bW
 cognite/neat/rules/issues/dms.py,sha256=CKztcpNu9E_ygbAmiODOhaYKPX6o9eaXeiod7Ak-kNY,23617
 cognite/neat/rules/issues/fileread.py,sha256=ao199mtvhPSW0IA8ZQZ0RzuLIIipYtL0jp6fLqxb4_c,5748
 cognite/neat/rules/issues/formatters.py,sha256=_ag2bJ9hncOj8pAGJvTTEPs9kTtxbD7vkqvS9Zcnizc,3385
-cognite/neat/rules/issues/importing.py,sha256=l0VKmOWN-qyvOMukNAVJZrN2WGD2YcGXeCKbxV-G_vA,12726
+cognite/neat/rules/issues/importing.py,sha256=uSk4TXo_CO3bglBZkaiWekXLXXd31UWIZE95clVSLz4,13417
 cognite/neat/rules/issues/spreadsheet.py,sha256=1NZQ3a_zCtErAoMKzEt5UMJsukaysoOVmW7WUF6JK1s,15825
 cognite/neat/rules/issues/spreadsheet_file.py,sha256=YCp0Pk_TsiqYuOPdWpjUpre-zvi2c5_MvrC_dxw10YY,4964
 cognite/neat/rules/models/__init__.py,sha256=aqhQUidHYgOk5_iqdi6s72s2g8qyMRFXShYzh-ctNpw,782
@@ -198,7 +198,7 @@ cognite/neat/rules/models/_base.py,sha256=oQ8f0bvdythJ2m54K1jl2OXEuEZ4N8JqHDXyhC
 cognite/neat/rules/models/_rdfpath.py,sha256=RoHnfWufjnDtwJh7UUzWKoJz8luvX7Gb5SDQORfkQTE,11030
 cognite/neat/rules/models/_types/__init__.py,sha256=l1tGxzE7ezNHIL72AoEvNHN2IFuitxOLxiHJG__s6t4,305
 cognite/neat/rules/models/_types/_base.py,sha256=2GhLUE1ukV8X8SGL_JDxpbWGZyAvOnSqAE6JmDh5wbI,929
-cognite/neat/rules/models/_types/_field.py,sha256=74WfCSVbTubpK4n4VsysQqCch6VI8IcPqnHQpvNbFZ8,3197
+cognite/neat/rules/models/_types/_field.py,sha256=h2RrhjxdaRbzHG5EyduyHLkCJJmQoyZb9pYCkgMcnVk,3203
 cognite/neat/rules/models/data_types.py,sha256=lanwkhwG8iHKfjYfia4v2SBTJrMeXOsqaVkVEP2QMXs,6078
 cognite/neat/rules/models/dms/__init__.py,sha256=Wzyqzz2ZIjpUbDg04CMuuIAw-f2A02DayNeqO9R-2Hw,491
 cognite/neat/rules/models/dms/_converter.py,sha256=QaJT0z0Yo4gkG9tHPZkU8idF8PK7c-tDahbyIT-WJQU,5959
@@ -207,11 +207,11 @@ cognite/neat/rules/models/dms/_rules.py,sha256=iqoPilY3tov3GvJ9N3K3go6xy9kiiMt9v
 cognite/neat/rules/models/dms/_rules_input.py,sha256=apDDTQll9UAyYL5gS2vDxHsujWrGBilTp7lK2kzJWO8,13467
 cognite/neat/rules/models/dms/_schema.py,sha256=A4z8CINmLQgWzHoScxejRPMRo40ngKlyp1gOdPto8yU,43808
 cognite/neat/rules/models/dms/_serializer.py,sha256=iqp2zyyf8jEcU-R3PERuN8nu248xIqyxiWj4owAn92g,6406
-cognite/neat/rules/models/dms/_validation.py,sha256=TUmBxfSB42lsgIKVIDPPbaTaJ8TKi-gTbayqgPAmnfs,13656
+cognite/neat/rules/models/dms/_validation.py,sha256=nPSyfM1vGZ7d9Uv_2vF2HvMetygtehXW7eNtPD6eW8E,13937
 cognite/neat/rules/models/domain.py,sha256=tkKcHvDXnZ5IkOr1wHiuNBtE1h8OCFmf6GZSqzHzxjI,2814
 cognite/neat/rules/models/entities.py,sha256=iBG84Jr1qQ7PvkMJUJzJ1oWApeONb1IACixdJSztUhk,16395
 cognite/neat/rules/models/information/__init__.py,sha256=HR6g8xgyU53U7Ck8pPdbT70817Q4NC1r1pCRq5SA8iw,291
-cognite/neat/rules/models/information/_converter.py,sha256=qJl95-gPz6MwKLZpHyY_CIqf-ebZalHWZyYnsqr_FWc,9960
+cognite/neat/rules/models/information/_converter.py,sha256=JN63_G5bygdL5WCz-q0_ygiU0NHkzUxm5mZ3WD8yUes,11029
 cognite/neat/rules/models/information/_rules.py,sha256=tdCjvAtqnFzEo2zcx-BZHmvjSs28gVun2wz8UaT-AOA,13268
 cognite/neat/rules/models/information/_rules_input.py,sha256=IZp_Wnrac0nVaHKth1YttWQOs-kULGKLMtngNQFY40A,10147
 cognite/neat/rules/models/information/_serializer.py,sha256=yti9I_xJruxrib66YIBInhze___Io-oPTQH6uWDumPE,3503
@@ -257,7 +257,7 @@ cognite/neat/workflows/steps/lib/current/__init__.py,sha256=c22IznGdCSNCpXCi_yon
 cognite/neat/workflows/steps/lib/current/graph_extractor.py,sha256=vW9UpJScx5dFVCSairpOdWRdBdLpkCt2kNh6litbF0o,5161
 cognite/neat/workflows/steps/lib/current/graph_loader.py,sha256=HfGg1HRZhbV58TFu89FTjKeUxGsbCYLeFJIQFDN_pQM,2341
 cognite/neat/workflows/steps/lib/current/graph_store.py,sha256=r7VTxdaz8jJQU7FJbnRDMxvEYbSAZFNMABhPyfNwiFk,6295
-cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=BWscuNvTu_u6QQSxkVz4X4A1wAHW2Kfku8lhgrIK51M,23886
+cognite/neat/workflows/steps/lib/current/rules_exporter.py,sha256=iFwzDWgUDBSPajaNAcXvu9pVw-jX66upvwyMXyTq7SE,23822
 cognite/neat/workflows/steps/lib/current/rules_importer.py,sha256=oGIBh9iFYP3w_K0kLpM-OKTw70R1Mw_opD62MGkCJlk,11451
 cognite/neat/workflows/steps/lib/current/rules_validator.py,sha256=LwF9lXlnuPOxDSsOMMTHBi2BHc5rk08IF5zahS9yQuw,4844
 cognite/neat/workflows/steps/lib/io/__init__.py,sha256=k7IPbIq3ey19oRc5sA_15F99-O6dxzqbm1LihGRRo5A,32
@@ -275,8 +275,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.77.6.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.77.6.dist-info/METADATA,sha256=XhW7PcK8bdv1WEsMS8Oehu1AapiwLD9PrbjOrXkV4yY,9316
-cognite_neat-0.77.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.77.6.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.77.6.dist-info/RECORD,,
+cognite_neat-0.77.8.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.77.8.dist-info/METADATA,sha256=fr3l3w0YmPmZwtA9G4rKyL5fVBvN8ThIeBQNtJtzLwk,9316
+cognite_neat-0.77.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.77.8.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.77.8.dist-info/RECORD,,