cognite-neat 0.81.1__py3-none-any.whl → 0.81.3__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.81.1"
+__version__ = "0.81.3"

cognite/neat/graph/loaders/_rdf2dms.py

@@ -22,6 +22,7 @@ from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
 from cognite.neat.utils.upload import UploadDiffsID
+from cognite.neat.utils.utils import create_sha256_hash
 
 from ._base import CDFLoader
 
@@ -217,9 +218,10 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 tracker.issue(error)
                 yield error
             for target in values:
+                external_id = f"{identifier}.{prop}.{target}"
                 yield dm.EdgeApply(
                     space=self.instance_space,
-                    external_id=f"{identifier}.{prop}.{target}",
+                    external_id=external_id if len(external_id) < 256 else create_sha256_hash(external_id),
                     type=edge.type,
                     start_node=dm.DirectRelationReference(self.instance_space, identifier),
                     end_node=dm.DirectRelationReference(self.instance_space, target),

cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py

@@ -21,7 +21,7 @@ from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
 from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
 from cognite.neat.legacy.rules.models.rules import Property, Rules
 from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.utils import create_sha256_hash, generate_exception_report
 
 if sys.version_info >= (3, 11):
     from datetime import UTC
@@ -683,9 +683,10 @@ def to_edge(self, data_model: DMSSchemaComponents, add_class_prefix: bool = Fals
                             stacklevel=2,
                         )
 
+                external_id = f"{self.external_id}.{edge_one_to_many}.{end_node_external_id}"
                 edge = EdgeApply(
                     space=data_model.views[view_id].space,
-                    external_id=f"{self.external_id}-{edge_one_to_many}-{end_node_external_id}",
+                    external_id=external_id if len(external_id) < 256 else create_sha256_hash(external_id),
                     type=(data_model.views[view_id].space, edge_type_id),
                     start_node=(data_model.views[view_id].space, self.external_id),
                     end_node=(data_model.views[view_id].space, end_node_external_id),

cognite/neat/rules/issues/dms.py

@@ -31,6 +31,7 @@ __all__ = [
     "ReverseRelationMissingOtherSideWarning",
     "NodeTypeFilterOnParentViewWarning",
     "MissingViewInModelWarning",
+    "ViewSizeWarning",
     "ChangingContainerError",
     "ChangingViewError",
 ]
@@ -44,6 +45,30 @@ class DMSSchemaError(NeatValidationError, ABC): ...
 class DMSSchemaWarning(ValidationWarning, ABC): ...
 
 
+@dataclass(frozen=True)
+class ViewSizeWarning(DMSSchemaWarning):
+    description = (
+        "The number of properties in the {view} view is {count} which is more than "
+        "the recommended limit of {limit} properties. This can lead to performance issues."
+    )
+    fix = "Reduce the size of the view"
+    error_name: ClassVar[str] = "ViewSizeWarning"
+
+    view_id: dm.ViewId
+    limit: int
+    count: int
+
+    def message(self) -> str:
+        return self.description.format(view=repr(self.view_id), count=self.count, limit=self.limit)
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        output["limit"] = self.limit
+        output["count"] = self.count
+        return output
+
+
 @dataclass(frozen=True)
 class IncompleteSchemaError(DMSSchemaError):
     description = "This error is raised when the schema is claimed to be complete but missing some components"

cognite/neat/rules/models/_constants.py

@@ -0,0 +1 @@
+DMS_CONTAINER_SIZE_LIMIT = 100

cognite/neat/rules/models/dms/_validation.py

@@ -6,6 +6,7 @@ from cognite.client import data_modeling as dm
 from cognite.neat.rules import issues
 from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
+from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ContainerEntity
 from cognite.neat.rules.models.wrapped_entities import RawFilter
@@ -34,7 +35,7 @@ class DMSPostValidation:
         self._validate_raw_filter()
         self._consistent_container_properties()
 
-        self._referenced_views_and_containers_are_existing()
+        self._referenced_views_and_containers_are_existing_and_proper_size()
         if self.metadata.schema_ is SchemaCompleteness.extended:
             self._validate_extension()
         if self.metadata.schema_ is SchemaCompleteness.partial:
@@ -118,14 +119,16 @@ class DMSPostValidation:
                 prop.constraint = prop.constraint or constraint_definition
         self.issue_list.extend(errors)
 
-    def _referenced_views_and_containers_are_existing(self) -> None:
+    def _referenced_views_and_containers_are_existing_and_proper_size(self) -> None:
         defined_views = {view.view.as_id() for view in self.views}
         if self.metadata.schema_ is SchemaCompleteness.extended and self.rules.last:
             defined_views |= {view.view.as_id() for view in self.rules.last.views}
 
-        errors: list[issues.NeatValidationError] = []
+        property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
+        errors: list[issues.ValidationIssue] = []
         for prop_no, prop in enumerate(self.properties):
-            if prop.view and (view_id := prop.view.as_id()) not in defined_views:
+            view_id = prop.view.as_id()
+            if view_id not in defined_views:
                 errors.append(
                     issues.spreadsheet.NonExistingViewError(
                         column="View",
@@ -137,6 +140,17 @@ class DMSPostValidation:
                         url=None,
                     )
                 )
+            else:
+                property_count_by_view[view_id] += 1
+        for view_id, count in property_count_by_view.items():
+            if count > DMS_CONTAINER_SIZE_LIMIT:
+                errors.append(
+                    issues.dms.ViewSizeWarning(
+                        view_id=view_id,
+                        limit=DMS_CONTAINER_SIZE_LIMIT,
+                        count=count,
+                    )
+                )
         if self.metadata.schema_ is SchemaCompleteness.complete:
             defined_containers = {container.container.as_id() for container in self.containers or []}
             if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:

cognite/neat/rules/models/information/_converter.py

@@ -8,6 +8,7 @@ from cognite.neat.rules.models._base import (
     SchemaCompleteness,
     SheetList,
 )
+from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.domain import DomainRules
 from cognite.neat.rules.models.entities import (
@@ -41,6 +42,7 @@ class _InformationRulesConverter:
             self.last_classes = {class_.class_: class_ for class_ in self.rules.last.classes}
         else:
             self.last_classes = {}
+        self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
 
     def as_domain_rules(self) -> DomainRules:
         raise NotImplementedError("DomainRules not implemented yet")
@@ -81,7 +83,6 @@ class _InformationRulesConverter:
         last_dms_rules = self.rules.last.as_dms_architect_rules() if self.rules.last else None
         ref_dms_rules = self.rules.reference.as_dms_architect_rules() if self.rules.reference else None
 
-        containers: list[DMSContainer] = []
         class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
         if self.rules.last:
             for cls_ in self.rules.last.classes:
@@ -93,6 +94,7 @@ class _InformationRulesConverter:
             if rule_set:
                 existing_containers.update({c.container for c in rule_set.containers or []})
 
+        containers: list[DMSContainer] = []
         for container_entity, class_entities in referenced_containers.items():
             if container_entity in existing_containers:
                 continue
@@ -227,9 +229,14 @@ class _InformationRulesConverter:
             # the existing container in the last schema
             container_entity = prop.class_.as_container_entity(default_space)
             container_entity.suffix = self._bump_suffix(container_entity.suffix)
-            return container_entity, prop.property_
         else:
-            return prop.class_.as_container_entity(default_space), prop.property_
+            container_entity = prop.class_.as_container_entity(default_space)
+
+        while self.property_count_by_container[container_entity] >= DMS_CONTAINER_SIZE_LIMIT:
+            container_entity.suffix = self._bump_suffix(container_entity.suffix)
+
+        self.property_count_by_container[container_entity] += 1
+        return container_entity, prop.property_
 
     def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
         if isinstance(cls_.reference, ReferenceEntity) and cls_.reference.prefix != metadata.prefix:

{cognite_neat-0.81.1.dist-info → cognite_neat-0.81.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.81.1
+Version: 0.81.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

{cognite_neat-0.81.1.dist-info → cognite_neat-0.81.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=v-rRiDOgZ3sQSMQKq0vgUQZvpeOkoHFXissAx6Ktg84,61
-cognite/neat/_version.py,sha256=zEgwrETiUxKyTUiAscaK4aVaaACC4OjGI69_cwGk124,23
+cognite/neat/_version.py,sha256=bK5LLQ5c0-mwXmJLM9_cE_lGPthPpVuCp0sGU3PavkI,23
 cognite/neat/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/app/api/asgi/metrics.py,sha256=nxFy7L5cChTI0a-zkCiJ59Aq8yLuIJp5c9Dg0wRXtV0,152
 cognite/neat/app/api/configuration.py,sha256=2U5M6M252swvQPQyooA1EBzFUZNtcTmuSaywfJDgckM,4232
@@ -70,7 +70,7 @@ cognite/neat/graph/issues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 cognite/neat/graph/issues/loader.py,sha256=v8YDsehkUT1QUG61JM9BDV_lqowMUnDmGmbay0aFzN4,3085
 cognite/neat/graph/loaders/__init__.py,sha256=hHC9sfFfbnGSVFTYeuNTIEu4tdLSJ2mWV07fereLelo,125
 cognite/neat/graph/loaders/_base.py,sha256=bdYC6CwsHVqnQa1QzOhL68qQhF1OtrsearqH6D-z3E4,4037
-cognite/neat/graph/loaders/_rdf2dms.py,sha256=Tn7vy6XwXFXpVDn7uzfzgJMJapbPITerKaF5b5Y4ol4,12857
+cognite/neat/graph/loaders/_rdf2dms.py,sha256=w0y2ECKw7RrQndGyTVIWeFF2WDxs9yvl_eWNqg-tKO8,13018
 cognite/neat/graph/models.py,sha256=AtLgZh2qyRP6NRetjQCy9qLMuTQB0CH52Zsev-qa2sk,149
 cognite/neat/graph/stores/__init__.py,sha256=G-VG_YwfRt1kuPao07PDJyZ3w_0-eguzLUM13n-Z_RA,64
 cognite/neat/graph/stores/_base.py,sha256=6MZAXygT6sHTQ1LWm_TDb2Ws6fgNJ-r4evwcLywpBVk,9481
@@ -135,7 +135,7 @@ cognite/neat/legacy/rules/exporters/_rules2dms.py,sha256=13CptTLvY9ghcrLPhumUOg6
 cognite/neat/legacy/rules/exporters/_rules2excel.py,sha256=ytHsqw2j26T9yLNZHuUSItV8Jp3AvvpIwX8D5-L9GO8,8312
 cognite/neat/legacy/rules/exporters/_rules2graphql.py,sha256=oXBU5z-qFyxG7MW83HYlW-hazhDDNAPKAbJJcsZfcU4,6251
 cognite/neat/legacy/rules/exporters/_rules2ontology.py,sha256=m6adoKOP5EVVEjFX4Qi9yw7UflrDRVgNiBxQ9QVgz6g,18458
-cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py,sha256=0Cn9juOgNPnmxLe7eUCUNjfaicLcQPpj_qA8HLFB3lc,28824
+cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py,sha256=YqiyTzAD0vEEhLbIPNTkK0AzKUG0yfBjz47Rg1mSTq8,28949
 cognite/neat/legacy/rules/exporters/_rules2rules.py,sha256=KlBm1hWkx4Ly5G-_gdcURUwADolMJFnueus02IW51uQ,3881
 cognite/neat/legacy/rules/exporters/_rules2triples.py,sha256=ItkLy6Rji4g5UqLtxaOeodGUvpQG-LVr_ss70PcCPZs,1085
 cognite/neat/legacy/rules/exporters/_validation.py,sha256=saDorwUqJ4Fo6yeCMSRH0Hp3AGCr-rdjb-sOGo91xL0,5767
@@ -204,7 +204,7 @@ cognite/neat/rules/importers/_spreadsheet2rules.py,sha256=nKSJyZGoTho0bqQ_5_1XB9
 cognite/neat/rules/importers/_yaml2rules.py,sha256=F0uksSz1A3po5OlRM2152_w5j8D9oYTLB9NFTkSMlWI,4275
 cognite/neat/rules/issues/__init__.py,sha256=c12m0HAHHzF6oR8lKbULE3TxOPimTi9s1O9IIrtgh0g,549
 cognite/neat/rules/issues/base.py,sha256=x2YLCfmqtPlFLoURq3qHaprXCpFaQdf0iWkql-EMyps,2446
-cognite/neat/rules/issues/dms.py,sha256=CKztcpNu9E_ygbAmiODOhaYKPX6o9eaXeiod7Ak-kNY,23617
+cognite/neat/rules/issues/dms.py,sha256=eZmbQhdo97SufNLKJu4QWlrhZCxmngiWTwPWIOx7GSA,24400
 cognite/neat/rules/issues/fileread.py,sha256=ao199mtvhPSW0IA8ZQZ0RzuLIIipYtL0jp6fLqxb4_c,5748
 cognite/neat/rules/issues/formatters.py,sha256=_ag2bJ9hncOj8pAGJvTTEPs9kTtxbD7vkqvS9Zcnizc,3385
 cognite/neat/rules/issues/importing.py,sha256=uSk4TXo_CO3bglBZkaiWekXLXXd31UWIZE95clVSLz4,13417
@@ -212,6 +212,7 @@ cognite/neat/rules/issues/spreadsheet.py,sha256=jBEczqon1G0H_mCfdCCffWdRLHO5ER8S
 cognite/neat/rules/issues/spreadsheet_file.py,sha256=YCp0Pk_TsiqYuOPdWpjUpre-zvi2c5_MvrC_dxw10YY,4964
 cognite/neat/rules/models/__init__.py,sha256=aqhQUidHYgOk5_iqdi6s72s2g8qyMRFXShYzh-ctNpw,782
 cognite/neat/rules/models/_base.py,sha256=7GUCflYZ7CDVyRZTYd4CYQJr7tPnMefd-1B9UktaWpY,11194
+cognite/neat/rules/models/_constants.py,sha256=zPREgHT79_4FMg58QlaXc7A8XKRJrjP5SUgh63jDnTk,31
 cognite/neat/rules/models/_rdfpath.py,sha256=RoHnfWufjnDtwJh7UUzWKoJz8luvX7Gb5SDQORfkQTE,11030
 cognite/neat/rules/models/_types/__init__.py,sha256=l1tGxzE7ezNHIL72AoEvNHN2IFuitxOLxiHJG__s6t4,305
 cognite/neat/rules/models/_types/_base.py,sha256=2GhLUE1ukV8X8SGL_JDxpbWGZyAvOnSqAE6JmDh5wbI,929
@@ -224,11 +225,11 @@ cognite/neat/rules/models/dms/_rules.py,sha256=XTIEWG49VjNs_bICGlgMd6uk4hseY1H6U
 cognite/neat/rules/models/dms/_rules_input.py,sha256=apDDTQll9UAyYL5gS2vDxHsujWrGBilTp7lK2kzJWO8,13467
 cognite/neat/rules/models/dms/_schema.py,sha256=byMG67i80a4sSQS_0k8YGrDvh7whio4iLbmPEIy_P44,49514
 cognite/neat/rules/models/dms/_serializer.py,sha256=iqp2zyyf8jEcU-R3PERuN8nu248xIqyxiWj4owAn92g,6406
-cognite/neat/rules/models/dms/_validation.py,sha256=nPSyfM1vGZ7d9Uv_2vF2HvMetygtehXW7eNtPD6eW8E,13937
+cognite/neat/rules/models/dms/_validation.py,sha256=5mk9L99FSwC8Ok7weEjnFJ_OZnmqMWUc6XFMTfkqfDw,14549
 cognite/neat/rules/models/domain.py,sha256=wZ-DeIPFnacbNlxSrRuLzUpnhHdTpzNc22z0sDfisi4,2880
 cognite/neat/rules/models/entities.py,sha256=lkLsKg8U3Xto30PCB85ScDpv2SPRVq1ukVEQHzH53_g,18868
 cognite/neat/rules/models/information/__init__.py,sha256=HR6g8xgyU53U7Ck8pPdbT70817Q4NC1r1pCRq5SA8iw,291
-cognite/neat/rules/models/information/_converter.py,sha256=JN63_G5bygdL5WCz-q0_ygiU0NHkzUxm5mZ3WD8yUes,11029
+cognite/neat/rules/models/information/_converter.py,sha256=r0a2uyzv8m82xzAkYt_-ZXdMN5u46SA_mn95Oo7ng-s,11424
 cognite/neat/rules/models/information/_rules.py,sha256=ZVTOn5fEB-AbrXL8A6SN9DwOmF9FhgyS7FzibrkT6ZM,13546
 cognite/neat/rules/models/information/_rules_input.py,sha256=xmcQQl2vBYSG_IbxOwb6x4CdN3nIg_TY2-3RAeGDYic,10418
 cognite/neat/rules/models/information/_serializer.py,sha256=yti9I_xJruxrib66YIBInhze___Io-oPTQH6uWDumPE,3503
@@ -293,8 +294,8 @@ cognite/neat/workflows/steps_registry.py,sha256=fkTX14ZA7_gkUYfWIlx7A1XbCidvqR23
 cognite/neat/workflows/tasks.py,sha256=dqlJwKAb0jlkl7abbY8RRz3m7MT4SK8-7cntMWkOYjw,788
 cognite/neat/workflows/triggers.py,sha256=_BLNplzoz0iic367u1mhHMHiUrCwP-SLK6_CZzfODX0,7071
 cognite/neat/workflows/utils.py,sha256=gKdy3RLG7ctRhbCRwaDIWpL9Mi98zm56-d4jfHDqP1E,453
-cognite_neat-0.81.1.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.81.1.dist-info/METADATA,sha256=HvhapkfdDjeI4wNtoR6rsyloUItiSR1R9D-6VbnoIiQ,9290
-cognite_neat-0.81.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_neat-0.81.1.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
-cognite_neat-0.81.1.dist-info/RECORD,,
+cognite_neat-0.81.3.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.81.3.dist-info/METADATA,sha256=OqByinJuu__bDSdYEfAoesZoqR16aVN-1kTkITzv1Gg,9290
+cognite_neat-0.81.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_neat-0.81.3.dist-info/entry_points.txt,sha256=61FPqiWb25vbqB0KI7znG8nsg_ibLHBvTjYnkPvNFso,50
+cognite_neat-0.81.3.dist-info/RECORD,,