cognite-neat 0.81.1__tar.gz → 0.81.3__tar.gz

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.1
+Version: 0.81.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.81.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.81.3"

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/cognite/neat/graph/loaders/_rdf2dms.py

@@ -22,6 +22,7 @@ from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
 from cognite.neat.utils.upload import UploadDiffsID
+from cognite.neat.utils.utils import create_sha256_hash
 
 from ._base import CDFLoader
 
@@ -217,9 +218,10 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 tracker.issue(error)
                 yield error
             for target in values:
+                external_id = f"{identifier}.{prop}.{target}"
                 yield dm.EdgeApply(
                     space=self.instance_space,
-                    external_id=f"{identifier}.{prop}.{target}",
+                    external_id=external_id if len(external_id) < 256 else create_sha256_hash(external_id),
                     type=edge.type,
                     start_node=dm.DirectRelationReference(self.instance_space, identifier),
                     end_node=dm.DirectRelationReference(self.instance_space, target),

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py

@@ -21,7 +21,7 @@ from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
 from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
 from cognite.neat.legacy.rules.models.rules import Property, Rules
 from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.utils import create_sha256_hash, generate_exception_report
 
 if sys.version_info >= (3, 11):
     from datetime import UTC
@@ -683,9 +683,10 @@ def to_edge(self, data_model: DMSSchemaComponents, add_class_prefix: bool = Fals
                             stacklevel=2,
                         )
 
+                external_id = f"{self.external_id}.{edge_one_to_many}.{end_node_external_id}"
                 edge = EdgeApply(
                     space=data_model.views[view_id].space,
-                    external_id=f"{self.external_id}-{edge_one_to_many}-{end_node_external_id}",
+                    external_id=external_id if len(external_id) < 256 else create_sha256_hash(external_id),
                     type=(data_model.views[view_id].space, edge_type_id),
                     start_node=(data_model.views[view_id].space, self.external_id),
                     end_node=(data_model.views[view_id].space, end_node_external_id),

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/cognite/neat/rules/issues/dms.py

@@ -31,6 +31,7 @@ __all__ = [
     "ReverseRelationMissingOtherSideWarning",
     "NodeTypeFilterOnParentViewWarning",
     "MissingViewInModelWarning",
+    "ViewSizeWarning",
     "ChangingContainerError",
     "ChangingViewError",
 ]
@@ -44,6 +45,30 @@ class DMSSchemaError(NeatValidationError, ABC): ...
 class DMSSchemaWarning(ValidationWarning, ABC): ...
 
 
+@dataclass(frozen=True)
+class ViewSizeWarning(DMSSchemaWarning):
+    description = (
+        "The number of properties in the {view} view is {count} which is more than "
+        "the recommended limit of {limit} properties. This can lead to performance issues."
+    )
+    fix = "Reduce the size of the view"
+    error_name: ClassVar[str] = "ViewSizeWarning"
+
+    view_id: dm.ViewId
+    limit: int
+    count: int
+
+    def message(self) -> str:
+        return self.description.format(view=repr(self.view_id), count=self.count, limit=self.limit)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["limit"] = self.limit
+        output["count"] = self.count
+        return output
+
+
 @dataclass(frozen=True)
 class IncompleteSchemaError(DMSSchemaError):
     description = "This error is raised when the schema is claimed to be complete but missing some components"

cognite_neat-0.81.3/cognite/neat/rules/models/_constants.py

@@ -0,0 +1 @@
+DMS_CONTAINER_SIZE_LIMIT = 100

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/cognite/neat/rules/models/dms/_validation.py

@@ -6,6 +6,7 @@ from cognite.client import data_modeling as dm
 from cognite.neat.rules import issues
 from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
+from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ContainerEntity
 from cognite.neat.rules.models.wrapped_entities import RawFilter
@@ -34,7 +35,7 @@ class DMSPostValidation:
         self._validate_raw_filter()
         self._consistent_container_properties()
 
-        self._referenced_views_and_containers_are_existing()
+        self._referenced_views_and_containers_are_existing_and_proper_size()
         if self.metadata.schema_ is SchemaCompleteness.extended:
             self._validate_extension()
         if self.metadata.schema_ is SchemaCompleteness.partial:
@@ -118,14 +119,16 @@ class DMSPostValidation:
                 prop.constraint = prop.constraint or constraint_definition
         self.issue_list.extend(errors)
 
-    def _referenced_views_and_containers_are_existing(self) -> None:
+    def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
         defined_views = {view.view.as_id() for view in self.views}
         if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
             defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
-        errors: list[issues.NeatValidationError] = []
+        property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
+        errors: list[issues.ValidationIssue] = []
         for prop_no, prop in enumerate(self.properties):
-            if prop.view and (view_id := prop.view.as_id()) not in defined_views:
+            view_id = prop.view.as_id()
+            if view_id not in defined_views:
                 errors.append(
                     issues.spreadsheet.NonExistingViewError(
                         column="View",
@@ -137,6 +140,17 @@ class DMSPostValidation:
                         url=None,
                     )
                 )
+            else:
+                property_count_by_view[view_id] += 1
+        for view_id, count in property_count_by_view.items():
+            if count > DMS_CONTAINER_SIZE_LIMIT:
+                errors.append(
+                    issues.dms.ViewSizeWarning(
+                        view_id=view_id,
+                        limit=DMS_CONTAINER_SIZE_LIMIT,
+                        count=count,
+                    )
+                )
         if self.metadata.schema_ is SchemaCompleteness.complete:
             defined_containers = {container.container.as_id() for container in self.containers or []}
             if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/cognite/neat/rules/models/information/_converter.py

@@ -8,6 +8,7 @@ from cognite.neat.rules.models._base import (
     SchemaCompleteness,
     SheetList,
 )
+from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.domain import DomainRules
 from cognite.neat.rules.models.entities import (
@@ -41,6 +42,7 @@ class _InformationRulesConverter:
             self.last_classes = {class_.class_: class_ for class_ in self.rules.last.classes}
         else:
             self.last_classes = {}
+        self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
 
     def as_domain_rules(self) -> DomainRules:
         raise NotImplementedError("DomainRules not implemented yet")
@@ -81,7 +83,6 @@ class _InformationRulesConverter:
         last_dms_rules = self.rules.last.as_dms_architect_rules() if self.rules.last else None
         ref_dms_rules = self.rules.reference.as_dms_architect_rules() if self.rules.reference else None
 
-        containers: list[DMSContainer] = []
         class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
         if self.rules.last:
             for cls_ in self.rules.last.classes:
@@ -93,6 +94,7 @@ class _InformationRulesConverter:
             if rule_set:
                 existing_containers.update({c.container for c in rule_set.containers or []})
 
+        containers: list[DMSContainer] = []
         for container_entity, class_entities in referenced_containers.items():
             if container_entity in existing_containers:
                 continue
@@ -227,9 +229,14 @@ class _InformationRulesConverter:
             # the existing container in the last schema
             container_entity = prop.class_.as_container_entity(default_space)
             container_entity.suffix = self._bump_suffix(container_entity.suffix)
-            return container_entity, prop.property_
         else:
-            return prop.class_.as_container_entity(default_space), prop.property_
+            container_entity = prop.class_.as_container_entity(default_space)
+
+        while self.property_count_by_container[container_entity] >= DMS_CONTAINER_SIZE_LIMIT:
+            container_entity.suffix = self._bump_suffix(container_entity.suffix)
+
+        self.property_count_by_container[container_entity] += 1
+        return container_entity, prop.property_
 
     def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
         if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:

{cognite_neat-0.81.1 → cognite_neat-0.81.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.81.1"
+version = "0.81.3"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.81.1/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.81.1"