cognite-neat 0.105.0__py3-none-any.whl → 0.105.1__py3-none-any.whl

cognite/neat/_constants.py

@@ -103,6 +103,7 @@ DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
 
 DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
 DMS_VIEW_CONTAINER_SIZE_LIMIT = 10
+DMS_DIRECT_RELATION_LIST_LIMIT = 100
 
 _ASSET_ROOT_PROPERTY = {
     "connection": "direct",

cognite/neat/_graph/loaders/_rdf2dms.py

@@ -1,10 +1,11 @@
 import itertools
 import json
+import warnings
 from collections import defaultdict
 from collections.abc import Iterable, Sequence
 from graphlib import TopologicalSorter
 from pathlib import Path
-from typing import Any, get_args
+from typing import Any, cast, get_args
 
 import yaml
 from cognite.client import CogniteClient
@@ -19,7 +20,7 @@ from pydantic import BaseModel, ValidationInfo, create_model, field_validator
 from rdflib import RDF, URIRef
 
 from cognite.neat._client import NeatClient
-from cognite.neat._constants import is_readonly_property
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT, is_readonly_property
 from cognite.neat._graph._tracking import LogTracker, Tracker
 from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
 from cognite.neat._issues.errors import (
@@ -28,7 +29,7 @@ from cognite.neat._issues.errors import (
     ResourceDuplicatedError,
     ResourceRetrievalError,
 )
-from cognite.neat._issues.warnings import PropertyTypeNotSupportedWarning
+from cognite.neat._issues.warnings import PropertyDirectRelationLimitWarning, PropertyTypeNotSupportedWarning
 from cognite.neat._rules.analysis._dms import DMSAnalysis
 from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
@@ -373,7 +374,21 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             def parse_direct_relation(cls, value: list, info: ValidationInfo) -> dict | list[dict]:
                 # We validate above that we only get one value for single direct relations.
                 if list.__name__ in _get_field_value_types(cls, info):
-                    return [{"space": self.instance_space, "externalId": remove_namespace_from_uri(v)} for v in value]
+                    result = [{"space": self.instance_space, "externalId": remove_namespace_from_uri(v)} for v in value]
+                    if len(result) <= DMS_DIRECT_RELATION_LIST_LIMIT:
+                        return result
+                    warnings.warn(
+                        PropertyDirectRelationLimitWarning(
+                            identifier="unknown",
+                            resource_type="view property",
+                            property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
+                            limit=DMS_DIRECT_RELATION_LIST_LIMIT,
+                        ),
+                        stacklevel=2,
+                    )
+                    # To get deterministic results, we sort by space and externalId
+                    result.sort(key=lambda x: (x["space"], x["externalId"]))
+                    return result[:DMS_DIRECT_RELATION_LIST_LIMIT]
                 elif value:
                     return {"space": self.instance_space, "externalId": remove_namespace_from_uri(value[0])}
                 return {}

cognite/neat/_graph/transformers/_classic_cdf.py

@@ -6,6 +6,7 @@ from functools import lru_cache
 from typing import cast
 
 from rdflib import RDF, Graph, Literal, Namespace, URIRef
+from rdflib.query import ResultRow
 
 from cognite.neat._constants import CLASSIC_CDF_NAMESPACE, DEFAULT_NAMESPACE
 from cognite.neat._graph import extractors
@@ -18,71 +19,57 @@ from cognite.neat._utils.rdf_ import (
     remove_namespace_from_uri,
 )
 
-from ._base import BaseTransformer
+from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
 
-# TODO: standardise
-class AddAssetDepth(BaseTransformer):
-    description: str = "Adds depth of asset in the asset hierarchy to the graph"
+class AddAssetDepth(BaseTransformerStandardised):
+    description: str = "Adds depth of asset in the asset hierarchy and optionally types asset based on depth"
     _use_only_once: bool = True
     _need_changes = frozenset({str(extractors.AssetsExtractor.__name__)})
 
-    _parent_template: str = """SELECT ?child ?parent WHERE {{
-                              <{asset_id}> <{parent_prop}> ?child .
-                              OPTIONAL{{?child <{parent_prop}>+ ?parent .}}}}"""
-
-    _root_template: str = """SELECT ?root WHERE {{
-                             <{asset_id}> <{root_prop}> ?root .}}"""
-
     def __init__(
         self,
         asset_type: URIRef | None = None,
-        root_prop: URIRef | None = None,
         parent_prop: URIRef | None = None,
         depth_typing: dict[int, str] | None = None,
     ):
         self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
-        self.root_prop = root_prop or DEFAULT_NAMESPACE.rootId
         self.parent_prop = parent_prop or DEFAULT_NAMESPACE.parentId
         self.depth_typing = depth_typing
 
-    def transform(self, graph: Graph) -> None:
-        """Adds depth of asset in the asset hierarchy to the graph."""
-        for result in graph.query(f"SELECT DISTINCT ?asset_id WHERE {{?asset_id a <{self.asset_type}>}}"):
-            asset_id = cast(tuple, result)[0]
-            if depth := self.get_depth(graph, asset_id, self.root_prop, self.parent_prop):
-                graph.add((asset_id, DEFAULT_NAMESPACE.depth, Literal(depth)))
-
-                if self.depth_typing and (type_ := self.depth_typing.get(depth, None)):
-                    # remove existing type
-                    graph.remove((asset_id, RDF.type, None))
-
-                    # add new type
-                    graph.add((asset_id, RDF.type, DEFAULT_NAMESPACE[type_]))
-
-    @classmethod
-    def get_depth(
-        cls,
-        graph: Graph,
-        asset_id: URIRef,
-        root_prop: URIRef,
-        parent_prop: URIRef,
-    ) -> int | None:
-        """Get asset depth in the asset hierarchy."""
-
-        # Handles non-root assets
-        if result := list(graph.query(cls._parent_template.format(asset_id=asset_id, parent_prop=parent_prop))):
-            return len(cast(list[tuple], result)) + 2 if cast(list[tuple], result)[0][1] else 2
-
-        # Handles root assets
-        elif (
-            (result := list(graph.query(cls._root_template.format(asset_id=asset_id, root_prop=root_prop))))
-            and len(cast(list[tuple], result)) == 1
-            and cast(list[tuple], result)[0][0] == asset_id
-        ):
-            return 1
-        else:
-            return None
+    def _iterate_query(self) -> str:
+        query = """SELECT ?asset (IF(?isRoot, 0, COUNT(?parent)) AS ?parentCount)
+                   WHERE {{
+                        ?asset a <{asset_type}> .
+                        OPTIONAL {{ ?asset <{parent_prop}>+ ?parent . }}
+                        BIND(IF(BOUND(?parent), false, true) AS ?isRoot)}}
+                   GROUP BY ?asset ?isRoot
+                   ORDER BY DESC(?parentCount)"""
+
+        return query.format(
+            asset_type=self.asset_type,
+            parent_prop=self.parent_prop,
+        )
+
+    def _count_query(self) -> str:
+        query = """SELECT (COUNT(?asset) as ?count)
+                   WHERE {{ ?asset a <{asset_type}> . }}"""
+
+        return query.format(asset_type=self.asset_type)
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+        subject, object = query_result_row
+
+        row_output.add_triples.append(cast(Triple, (subject, DEFAULT_NAMESPACE.depth, object)))
+
+        if self.depth_typing and (type_ := self.depth_typing.get(int(object), None)):
+            row_output.remove_triples.append(cast(Triple, (subject, RDF.type, self.asset_type)))
+            row_output.add_triples.append(cast(Triple, (subject, RDF.type, DEFAULT_NAMESPACE[type_])))
+
+        row_output.instances_modified_count += 1
+
+        return row_output
 
 
 # TODO: standardise

cognite/neat/_graph/transformers/_rdfpath.py

@@ -1,15 +1,14 @@
 from typing import cast
 from urllib.parse import quote
 
-from rdflib import Graph, URIRef
+from rdflib import Graph, Namespace, URIRef
 from rdflib.query import ResultRow
 
-from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.analysis import InformationAnalysis
 from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
 from cognite.neat._rules.models.information import InformationRules
 from cognite.neat._shared import Triple
-from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
 
 from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
@@ -76,11 +75,11 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
         self.subject_predicate = subject_predicate
         self.object_type = object_type
         self.object_predicate = object_predicate
-
+        subject_namespace = Namespace(get_namespace(subject_type))
         self.connection = (
-            DEFAULT_NAMESPACE[quote(connection.strip())]
+            subject_namespace[quote(connection.strip())]
             if isinstance(connection, str)
-            else connection or DEFAULT_NAMESPACE[remove_namespace_from_uri(self.object_type).lower()]
+            else connection or subject_namespace[remove_namespace_from_uri(self.object_type).lower()]
         )
 
         self.limit = limit
@@ -88,10 +87,10 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
     def _iterate_query(self) -> str:
         query = """SELECT DISTINCT ?subject ?object
                             WHERE {{
-                                ?subject a <{subject_type}> .
-                                ?subject <{subject_predicate}> ?value .
-                                ?object <{object_predicate}> ?value .
-                                ?object a <{object_type}> .
+                                ?subject a <{subject_type}> ;
+                                        <{subject_predicate}> ?value .
+                                ?object a <{object_type}> ;
+                                        <{object_predicate}> ?value .
                             }}"""
 
         if self.limit and isinstance(self.limit, int) and self.limit > 0:
@@ -105,12 +104,12 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
         )
 
     def _count_query(self) -> str:
-        query = """SELECT (COUNT(DISTINCT (?subject ?object)) as ?count)
+        query = """SELECT (COUNT(DISTINCT ?subject) as ?count)
                     WHERE {{
-                        ?subject a <{subject_type}> .
-                        ?subject <{subject_predicate}> ?value .
-                        ?object <{object_predicate}> ?value .
-                        ?object a <{object_type}> .
+                        ?subject a <{subject_type}> ;
+                                <{subject_predicate}> ?value .
+                        ?object a <{object_type}> ;
+                                <{object_predicate}> ?value .
                     }}"""
 
         if self.limit and isinstance(self.limit, int) and self.limit > 0:

cognite/neat/_issues/warnings/__init__.py

@@ -28,6 +28,7 @@ from ._models import (
 from ._properties import (
     PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
+    PropertyDirectRelationLimitWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
     PropertySkippedWarning,
@@ -64,6 +65,7 @@ __all__ = [
     "PrincipleSolutionBuildsOnEnterpriseWarning",
     "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
+    "PropertyDirectRelationLimitWarning",
     "PropertyNotFoundWarning",
     "PropertyOverwritingWarning",
     "PropertySkippedWarning",

cognite/neat/_issues/warnings/_properties.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Generic
 
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT
 from cognite.neat._issues._base import ResourceType
 
 from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
@@ -77,3 +78,13 @@ class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
     """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""
 
     error: str
+
+
+@dataclass(unsafe_hash=True)
+class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
+    """The listable direct relation property {property_name} in the {resource_type} with identifier {identifier}
+    has more than {limit} relations. The relations will be sorted by (space, externalId) and truncated."""
+
+    resource_type = "view"
+
+    limit: int = DMS_DIRECT_RELATION_LIST_LIMIT

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.105.0"
+__version__ = "0.105.1"
 __engine__ = "^2.0.3"

{cognite_neat-0.105.0.dist-info → cognite_neat-0.105.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.105.0
+Version: 0.105.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -20,7 +20,7 @@ Provides-Extra: oxi
 Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.71.2,<8.0.0)
 Requires-Dist: elementpath (>=4.0.0,<5.0.0)
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
 Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"

{cognite_neat-0.105.0.dist-info → cognite_neat-0.105.1.dist-info}/RECORD

@@ -10,7 +10,7 @@ cognite/neat/_client/data_classes/neat_sequence.py,sha256=QZWSfWnwk6KlYJvsInco4W
 cognite/neat/_client/data_classes/schema.py,sha256=uD8ExxEiIP3zhK4b--Q5fND-vmcC05a9WU5ItLsqG88,23179
 cognite/neat/_client/testing.py,sha256=c5ADJkRJFYGlJVQz-uPqxKExKXT297pxKh_ka4oGBjs,1082
 cognite/neat/_config.py,sha256=f9Py4SEHwYYquIg-k1rC7MbXBLENXQauoZtLyUbWvJQ,10118
-cognite/neat/_constants.py,sha256=XO5-wL6wjY722MQAmWfIW-sC0WT_PTWSFL_GpYSRFjQ,4962
+cognite/neat/_constants.py,sha256=ei0ODx_P1Z4O-8tVK_K_7oRvEMzovVJ-0kw96fW8a_0,4999
 cognite/neat/_graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/_graph/_shared.py,sha256=g7XFITbVxdDyGZ6mlgFUv5cBycrU7QbPktRikdUVkks,863
 cognite/neat/_graph/_tracking/__init__.py,sha256=WOwsYieZtCW-iW15YkxUFrfKVVdLWdXHOGGStTwvE8A,91
@@ -40,17 +40,17 @@ cognite/neat/_graph/extractors/_mock_graph_generator.py,sha256=7WqyFu2Qj03pJD4au
 cognite/neat/_graph/extractors/_rdf_file.py,sha256=YgPZN4Ayk6UlbwFFjdWn4Yo3P74D8KeNUb3slXg6Ox8,1604
 cognite/neat/_graph/loaders/__init__.py,sha256=1eam_rG1BXTUJ8iDm8_IYZldEe177vn2GmHihDBi8qk,718
 cognite/neat/_graph/loaders/_base.py,sha256=Fp6uUkNfAM-SVgsLz7tyNJxJ1eeEw3h2d4Q0YyppR-Y,3991
-cognite/neat/_graph/loaders/_rdf2dms.py,sha256=tTEGh_d0WZ_sTbqSkKUkb8IEtamiNGY46B5lnq0pCwc,24349
+cognite/neat/_graph/loaders/_rdf2dms.py,sha256=BskCCo_2P4JjPOyAc4His6tv_nEPH8u0qoQZyaSjVGQ,25247
 cognite/neat/_graph/queries/__init__.py,sha256=BgDd-037kvtWwAoGAy8eORVNMiZ5-E9sIV0txIpeaN4,50
 cognite/neat/_graph/queries/_base.py,sha256=Y3Amuave6xdQsDE5ZXCrYLUgOMIH8BXa4n5Pc9czAF0,14926
 cognite/neat/_graph/queries/_construct.py,sha256=CW8uHtXXACUXDj1AcEjROXtvoiuyx0CTgZ0bURY5Neo,7213
 cognite/neat/_graph/queries/_shared.py,sha256=uhw-nY4jJvivgtj1msdCRrfTWgauU7ybSHUqqUaFOUU,5390
 cognite/neat/_graph/transformers/__init__.py,sha256=sSrjK3_UDeesN8RzOjfNAYqyOEwjZJPUGO4uDuMLZ3s,1620
 cognite/neat/_graph/transformers/_base.py,sha256=omFmfRGaudojjq3GlW6b8PA2TNRcN3jXkw6Xvnx5r4M,4345
-cognite/neat/_graph/transformers/_classic_cdf.py,sha256=WoxjPNmuiLwIvbdVqeUX2HIjPoaBI4zWriR5Po1iXA0,19406
+cognite/neat/_graph/transformers/_classic_cdf.py,sha256=WaujTo8ryLSp7mZkyRdmAXHZ-AhgbOyM0X96S0rNhXs,18919
 cognite/neat/_graph/transformers/_iodd.py,sha256=KNz1fPdKK40UaHgPMECzNZgSeU5PdPRyLdaRYdq1iug,866
 cognite/neat/_graph/transformers/_prune_graph.py,sha256=LFiAMYFteND5LGEv9KqYJr5C9-n7S5fR6IrEdtJyRnk,12447
-cognite/neat/_graph/transformers/_rdfpath.py,sha256=jaJwtvfY65X2aRTJ__MSWcv_i_CyvOrlYYLRMCx2Ji4,4993
+cognite/neat/_graph/transformers/_rdfpath.py,sha256=WzydQvBzsnZ4BSv3HJWSH73e-sf1p-R4M2dm2H-DtEk,5019
 cognite/neat/_graph/transformers/_value_type.py,sha256=ZtH1KZmOGBGpUcqj4fBRlPOMq6kADt-HMShvYPS5Org,12841
 cognite/neat/_issues/__init__.py,sha256=OVgWivp_Br31p8zPeHjOEXs-Wj7lJU1pU1L3pfhfuqE,518
 cognite/neat/_issues/_base.py,sha256=vV0E8cfXKlOnRXFIDZKg5QPMruELEbCaUfQ01l5dR9A,20073
@@ -61,11 +61,11 @@ cognite/neat/_issues/errors/_properties.py,sha256=T_nquQeEQS3DQY--DQ13acxhGX_-gp
 cognite/neat/_issues/errors/_resources.py,sha256=YoajFF4Nxq_mhhVSZ7r3J6V-sH8cMMn576dSTsbcQZk,3964
 cognite/neat/_issues/errors/_workflow.py,sha256=m_Hlsvl5A1Oy7P3IROnz-4_do8_orZ1Pr1IHqsMyEys,971
 cognite/neat/_issues/formatters.py,sha256=ziNWT_YXwovTfU8Av5iYuSLgszzJYKTawM_z67VBdlU,3331
-cognite/neat/_issues/warnings/__init__.py,sha256=Qq5z9Bi_4Htn0bKeHQyxssfwHRSMv9nuQbJX1psWThI,2885
+cognite/neat/_issues/warnings/__init__.py,sha256=6iIa8IFb7i1TtCS6Bp2ZiwPggQPWziKv9_jsb9v1m0U,2967
 cognite/neat/_issues/warnings/_external.py,sha256=3tE98nLzOx9pb-VMi0MmQskHj-IuEqwjjoqhKMJ-xIM,1325
 cognite/neat/_issues/warnings/_general.py,sha256=idZZZDbeSrDJJ1PomdmIO40QsZQNn_lWztWvMA-9q50,782
 cognite/neat/_issues/warnings/_models.py,sha256=i4ZXr1IINKbFiVhUd8-qAt9_cXB8D3W-ng1Ime_lQTA,4376
-cognite/neat/_issues/warnings/_properties.py,sha256=xo_kaXaMOozmhxBRHhVTPaYobFpHWvfiO9q0UYqnKS8,2678
+cognite/neat/_issues/warnings/_properties.py,sha256=cC1mWcHm7NM2avheTh3eUZmNrnI-s5rqjGvg0-YR6NI,3146
 cognite/neat/_issues/warnings/_resources.py,sha256=jRhV7ROxuqcwah4rB3vTjBf_cZufRgnkDKZAflhlV3c,3556
 cognite/neat/_issues/warnings/user_modeling.py,sha256=_cN1wbaQUStZ13aG0VbN5UwKM9YdtyPfuSNJ1AAS6o8,3668
 cognite/neat/_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -170,10 +170,10 @@ cognite/neat/_utils/text.py,sha256=0IffvBIAmeGh92F4T6xiEdd-vv3B7FOGEMbfuTktO5Y,4
 cognite/neat/_utils/time_.py,sha256=O30LUiDH9TdOYz8_a9pFqTtJdg8vEjC3qHCk8xZblG8,345
 cognite/neat/_utils/upload.py,sha256=iWKmsQgw4EHLv-11NjYu7zAj5LtqTAfNa87a1kWeuaU,5727
 cognite/neat/_utils/xml_.py,sha256=FQkq84u35MUsnKcL6nTMJ9ajtG9D5i1u4VBnhGqP2DQ,1710
-cognite/neat/_version.py,sha256=k6808j5bC3aQEf0yqHcziCekV2_7uiJmkn-B2vMfJGs,46
+cognite/neat/_version.py,sha256=pwMGasqtYdWAmthqoS-VAnOaio6xE8H0G5lpVVh6afU,46
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite_neat-0.105.0.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.105.0.dist-info/METADATA,sha256=VocFhvXyWxYiP10dCC4xlD7SUyNq37jSH5BX6FqIPUA,5759
-cognite_neat-0.105.0.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-cognite_neat-0.105.0.dist-info/entry_points.txt,sha256=SsQlnl8SNMSSjE3acBI835JYFtsIinLSbVmHmMEXv6E,51
-cognite_neat-0.105.0.dist-info/RECORD,,
+cognite_neat-0.105.1.dist-info/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.105.1.dist-info/METADATA,sha256=Heyqxvr0rhVMEuHNlg8mphs00qmClzStLwhbdQBgPrk,5759
+cognite_neat-0.105.1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+cognite_neat-0.105.1.dist-info/entry_points.txt,sha256=SsQlnl8SNMSSjE3acBI835JYFtsIinLSbVmHmMEXv6E,51
+cognite_neat-0.105.1.dist-info/RECORD,,