cognite-neat 0.105.0__tar.gz → 0.105.1__tar.gz

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.105.0
+Version: 0.105.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -20,7 +20,7 @@ Provides-Extra: oxi
 Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.71.2,<8.0.0)
 Requires-Dist: elementpath (>=4.0.0,<5.0.0)
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
 Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_constants.py

@@ -103,6 +103,7 @@ DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"
 
 DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
 DMS_VIEW_CONTAINER_SIZE_LIMIT = 10
+DMS_DIRECT_RELATION_LIST_LIMIT = 100
 
 _ASSET_ROOT_PROPERTY = {
     "connection": "direct",

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_graph/loaders/_rdf2dms.py

@@ -1,10 +1,11 @@
 import itertools
 import json
+import warnings
 from collections import defaultdict
 from collections.abc import Iterable, Sequence
 from graphlib import TopologicalSorter
 from pathlib import Path
-from typing import Any, get_args
+from typing import Any, cast, get_args
 
 import yaml
 from cognite.client import CogniteClient
@@ -19,7 +20,7 @@ from pydantic import BaseModel, ValidationInfo, create_model, field_validator
 from rdflib import RDF, URIRef
 
 from cognite.neat._client import NeatClient
-from cognite.neat._constants import is_readonly_property
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT, is_readonly_property
 from cognite.neat._graph._tracking import LogTracker, Tracker
 from cognite.neat._issues import IssueList, NeatIssue, NeatIssueList
 from cognite.neat._issues.errors import (
@@ -28,7 +29,7 @@ from cognite.neat._issues.errors import (
     ResourceDuplicatedError,
     ResourceRetrievalError,
 )
-from cognite.neat._issues.warnings import PropertyTypeNotSupportedWarning
+from cognite.neat._issues.warnings import PropertyDirectRelationLimitWarning, PropertyTypeNotSupportedWarning
 from cognite.neat._rules.analysis._dms import DMSAnalysis
 from cognite.neat._rules.models import DMSRules
 from cognite.neat._rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
@@ -373,7 +374,21 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             def parse_direct_relation(cls, value: list, info: ValidationInfo) -> dict | list[dict]:
                 # We validate above that we only get one value for single direct relations.
                 if list.__name__ in _get_field_value_types(cls, info):
-                    return [{"space": self.instance_space, "externalId": remove_namespace_from_uri(v)} for v in value]
+                    result = [{"space": self.instance_space, "externalId": remove_namespace_from_uri(v)} for v in value]
+                    if len(result) <= DMS_DIRECT_RELATION_LIST_LIMIT:
+                        return result
+                    warnings.warn(
+                        PropertyDirectRelationLimitWarning(
+                            identifier="unknown",
+                            resource_type="view property",
+                            property_name=cast(str, cls.model_fields[info.field_name].alias or info.field_name),
+                            limit=DMS_DIRECT_RELATION_LIST_LIMIT,
+                        ),
+                        stacklevel=2,
+                    )
+                    # To get deterministic results, we sort by space and externalId
+                    result.sort(key=lambda x: (x["space"], x["externalId"]))
+                    return result[:DMS_DIRECT_RELATION_LIST_LIMIT]
                 elif value:
                     return {"space": self.instance_space, "externalId": remove_namespace_from_uri(value[0])}
                 return {}

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_graph/transformers/_classic_cdf.py

@@ -6,6 +6,7 @@ from functools import lru_cache
 from typing import cast
 
 from rdflib import RDF, Graph, Literal, Namespace, URIRef
+from rdflib.query import ResultRow
 
 from cognite.neat._constants import CLASSIC_CDF_NAMESPACE, DEFAULT_NAMESPACE
 from cognite.neat._graph import extractors
@@ -18,71 +19,57 @@ from cognite.neat._utils.rdf_ import (
     remove_namespace_from_uri,
 )
 
-from ._base import BaseTransformer
+from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
 
-# TODO: standardise
-class AddAssetDepth(BaseTransformer):
-    description: str = "Adds depth of asset in the asset hierarchy to the graph"
+class AddAssetDepth(BaseTransformerStandardised):
+    description: str = "Adds depth of asset in the asset hierarchy and optionally types asset based on depth"
     _use_only_once: bool = True
     _need_changes = frozenset({str(extractors.AssetsExtractor.__name__)})
 
-    _parent_template: str = """SELECT ?child ?parent WHERE {{
-                              <{asset_id}> <{parent_prop}> ?child .
-                              OPTIONAL{{?child <{parent_prop}>+ ?parent .}}}}"""
-
-    _root_template: str = """SELECT ?root WHERE {{
-                             <{asset_id}> <{root_prop}> ?root .}}"""
-
     def __init__(
         self,
         asset_type: URIRef | None = None,
-        root_prop: URIRef | None = None,
         parent_prop: URIRef | None = None,
         depth_typing: dict[int, str] | None = None,
     ):
         self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
-        self.root_prop = root_prop or DEFAULT_NAMESPACE.rootId
         self.parent_prop = parent_prop or DEFAULT_NAMESPACE.parentId
         self.depth_typing = depth_typing
 
-    def transform(self, graph: Graph) -> None:
-        """Adds depth of asset in the asset hierarchy to the graph."""
-        for result in graph.query(f"SELECT DISTINCT ?asset_id WHERE {{?asset_id a <{self.asset_type}>}}"):
-            asset_id = cast(tuple, result)[0]
-            if depth := self.get_depth(graph, asset_id, self.root_prop, self.parent_prop):
-                graph.add((asset_id, DEFAULT_NAMESPACE.depth, Literal(depth)))
-
-                if self.depth_typing and (type_ := self.depth_typing.get(depth, None)):
-                    # remove existing type
-                    graph.remove((asset_id, RDF.type, None))
-
-                    # add new type
-                    graph.add((asset_id, RDF.type, DEFAULT_NAMESPACE[type_]))
-
-    @classmethod
-    def get_depth(
-        cls,
-        graph: Graph,
-        asset_id: URIRef,
-        root_prop: URIRef,
-        parent_prop: URIRef,
-    ) -> int | None:
-        """Get asset depth in the asset hierarchy."""
-
-        # Handles non-root assets
-        if result := list(graph.query(cls._parent_template.format(asset_id=asset_id, parent_prop=parent_prop))):
-            return len(cast(list[tuple], result)) + 2 if cast(list[tuple], result)[0][1] else 2
-
-        # Handles root assets
-        elif (
-            (result := list(graph.query(cls._root_template.format(asset_id=asset_id, root_prop=root_prop))))
-            and len(cast(list[tuple], result)) == 1
-            and cast(list[tuple], result)[0][0] == asset_id
-        ):
-            return 1
-        else:
-            return None
+    def _iterate_query(self) -> str:
+        query = """SELECT ?asset (IF(?isRoot, 0, COUNT(?parent)) AS ?parentCount)
+                   WHERE {{
+                        ?asset a <{asset_type}> .
+                        OPTIONAL {{ ?asset <{parent_prop}>+ ?parent . }}
+                        BIND(IF(BOUND(?parent), false, true) AS ?isRoot)}}
+                   GROUP BY ?asset ?isRoot
+                   ORDER BY DESC(?parentCount)"""
+
+        return query.format(
+            asset_type=self.asset_type,
+            parent_prop=self.parent_prop,
+        )
+
+    def _count_query(self) -> str:
+        query = """SELECT (COUNT(?asset) as ?count)
+                   WHERE {{ ?asset a <{asset_type}> . }}"""
+
+        return query.format(asset_type=self.asset_type)
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+        subject, object = query_result_row
+
+        row_output.add_triples.append(cast(Triple, (subject, DEFAULT_NAMESPACE.depth, object)))
+
+        if self.depth_typing and (type_ := self.depth_typing.get(int(object), None)):
+            row_output.remove_triples.append(cast(Triple, (subject, RDF.type, self.asset_type)))
+            row_output.add_triples.append(cast(Triple, (subject, RDF.type, DEFAULT_NAMESPACE[type_])))
+
+        row_output.instances_modified_count += 1
+
+        return row_output
 
 
 # TODO: standardise

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_graph/transformers/_rdfpath.py

@@ -1,15 +1,14 @@
 from typing import cast
 from urllib.parse import quote
 
-from rdflib import Graph, URIRef
+from rdflib import Graph, Namespace, URIRef
 from rdflib.query import ResultRow
 
-from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._rules.analysis import InformationAnalysis
 from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
 from cognite.neat._rules.models.information import InformationRules
 from cognite.neat._shared import Triple
-from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
 
 from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
 
@@ -76,11 +75,11 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
         self.subject_predicate = subject_predicate
         self.object_type = object_type
         self.object_predicate = object_predicate
-
+        subject_namespace = Namespace(get_namespace(subject_type))
         self.connection = (
-            DEFAULT_NAMESPACE[quote(connection.strip())]
+            subject_namespace[quote(connection.strip())]
             if isinstance(connection, str)
-            else connection or DEFAULT_NAMESPACE[remove_namespace_from_uri(self.object_type).lower()]
+            else connection or subject_namespace[remove_namespace_from_uri(self.object_type).lower()]
         )
 
         self.limit = limit
@@ -88,10 +87,10 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
     def _iterate_query(self) -> str:
         query = """SELECT DISTINCT ?subject ?object
                             WHERE {{
-                                ?subject a <{subject_type}> .
-                                ?subject <{subject_predicate}> ?value .
-                                ?object <{object_predicate}> ?value .
-                                ?object a <{object_type}> .
+                                ?subject a <{subject_type}> ;
+                                        <{subject_predicate}> ?value .
+                                ?object a <{object_type}> ;
+                                        <{object_predicate}> ?value .
                             }}"""
 
         if self.limit and isinstance(self.limit, int) and self.limit > 0:
@@ -105,12 +104,12 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
         )
 
     def _count_query(self) -> str:
-        query = """SELECT (COUNT(DISTINCT (?subject ?object)) as ?count)
+        query = """SELECT (COUNT(DISTINCT ?subject) as ?count)
                     WHERE {{
-                        ?subject a <{subject_type}> .
-                        ?subject <{subject_predicate}> ?value .
-                        ?object <{object_predicate}> ?value .
-                        ?object a <{object_type}> .
+                        ?subject a <{subject_type}> ;
+                                <{subject_predicate}> ?value .
+                        ?object a <{object_type}> ;
+                                <{object_predicate}> ?value .
                     }}"""
 
         if self.limit and isinstance(self.limit, int) and self.limit > 0:

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_issues/warnings/__init__.py

@@ -28,6 +28,7 @@ from ._models import (
 from ._properties import (
     PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
+    PropertyDirectRelationLimitWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
     PropertySkippedWarning,
@@ -64,6 +65,7 @@ __all__ = [
     "PrincipleSolutionBuildsOnEnterpriseWarning",
     "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
+    "PropertyDirectRelationLimitWarning",
     "PropertyNotFoundWarning",
     "PropertyOverwritingWarning",
     "PropertySkippedWarning",

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/cognite/neat/_issues/warnings/_properties.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Generic
 
+from cognite.neat._constants import DMS_DIRECT_RELATION_LIST_LIMIT
 from cognite.neat._issues._base import ResourceType
 
 from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
@@ -77,3 +78,13 @@ class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
     """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""
 
     error: str
+
+
+@dataclass(unsafe_hash=True)
+class PropertyDirectRelationLimitWarning(PropertyWarning[T_Identifier]):
+    """The listable direct relation property {property_name} in the {resource_type} with identifier {identifier}
+    has more than {limit} relations. The relations will be sorted by (space, externalId) and truncated."""
+
+    resource_type = "view"
+
+    limit: int = DMS_DIRECT_RELATION_LIST_LIMIT

cognite_neat-0.105.1/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.105.1"
+__engine__ = "^2.0.3"

{cognite_neat-0.105.0 → cognite_neat-0.105.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.105.0"
+version = "0.105.1"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [
@@ -53,7 +53,7 @@ neat = "cognite.neat._app.main:run"
 python = "^3.10"
 # Core dependencies
 pandas = "*"
-cognite-sdk = "^7.54.6"
+cognite-sdk = "^7.71.2"
 rdflib = "*"
 pydantic = "^2"
 PyYAML = "*"
@@ -167,7 +167,7 @@ types-openpyxl = "^3"
 types-requests = "^2"
 types-setuptools = "^68"
 networkx-stubs = "^0.0.1"
-cognite-toolkit = "^0.3.23"
+cognite-toolkit = "^0.3.24"
 
 
 [build-system]

cognite_neat-0.105.0/cognite/neat/_version.py

@@ -1,2 +0,0 @@
-__version__ = "0.105.0"
-__engine__ = "^2.0.3"