cognite-neat 0.123.6__py3-none-any.whl → 0.123.8__py3-none-any.whl

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.123.6"
+__version__ = "0.123.8"
 __engine__ = "^2.0.4"

cognite/neat/core/_client/_api/data_modeling_loaders.py

@@ -1,8 +1,10 @@
+import re
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from collections.abc import Callable, Collection, Iterable, Sequence
 from dataclasses import dataclass, field
-from graphlib import TopologicalSorter
+from graphlib import CycleError, TopologicalSorter
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeVar, cast, overload
 
 from cognite.client.data_classes import filters
@@ -56,6 +58,7 @@ from cognite.neat.core._client.data_classes.data_modeling import Component
 from cognite.neat.core._client.data_classes.schema import DMSSchema
 from cognite.neat.core._issues.warnings import CDFMaxIterationsWarning
 from cognite.neat.core._shared import T_ID
+from cognite.neat.core._utils.tarjan import tarjan
 
 if TYPE_CHECKING:
     from cognite.neat.core._client._api_client import NeatClient
@@ -620,6 +623,81 @@ class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, View
     def _create(self, items: Sequence[ViewApply]) -> ViewList:
         return self._client.data_modeling.views.apply(items)
 
+    def create(self, items: Sequence[ViewApply]) -> ViewList:
+        try:
+            return self._create(items)
+        except CogniteAPIError as e1:
+            if self._is_auto_retryable(e1):
+                # Fallback to creating one by one if the error is auto-retryable.
+                return self._fallback_one_by_one(self._create, items)
+            elif self._is_false_not_exists(e1, {item.as_id() for item in items}):
+                return self._try_to_recover_coupled(items, e1)
+            raise
+
+    def _try_to_recover_coupled(self, items: Sequence[ViewApply], original_error: CogniteAPIError) -> ViewList:
+        """The /models/views endpoint can give faulty 400 about missing views that are part of the request.
+
+        This method tries to recover from such errors by identifying the strongly connected components in the graph
+        defined by the implements and through properties of the views. We then create the components in topological
+        order.
+
+        Args:
+            items: The items that failed to create.
+            original_error: The original error that was raised. If the problem is not recoverable, this error is raised.
+
+        Returns:
+            The views that were created.
+
+        """
+        views_by_id = {self.get_id(item): item for item in items}
+        parents_by_child: dict[ViewId, set[ViewId]] = {
+            view_id: {parent for parent in view.implements or [] if parent in views_by_id}
+            for view_id, view in views_by_id.items()
+        }
+        # Check for cycles in the implements graph
+        try:
+            TopologicalSorter(parents_by_child).static_order()
+        except CycleError as e:
+            raise CycleError(f"Failed to deploy views. This likely due to a cycle in implements. {e.args[1]}") from None
+
+        dependencies_by_id: dict[ViewId, set[ViewId]] = defaultdict(set)
+        for view_id, view in views_by_id.items():
+            dependencies_by_id[view_id].update([parent for parent in view.implements or [] if parent in views_by_id])
+            for properties in (view.properties or {}).values():
+                if isinstance(properties, ReverseDirectRelationApply):
+                    if isinstance(properties.through.source, ViewId) and properties.through.source in views_by_id:
+                        dependencies_by_id[view_id].add(properties.through.source)
+
+        created = ViewList([])
+        for strongly_connected in tarjan(dependencies_by_id):
+            to_create = [views_by_id[view_id] for view_id in strongly_connected]
+            try:
+                created_set = self._client.data_modeling.views.apply(to_create)
+            except CogniteAPIError:
+                self._client.data_modeling.views.delete(created.as_ids())
+                raise original_error from None
+            created.extend(created_set)
+        return created
+
+    @staticmethod
+    def _is_auto_retryable(e: CogniteAPIError) -> bool:
+        return isinstance(e.extra, dict) and "isAutoRetryable" in e.extra and e.extra["isAutoRetryable"]
+
+    @staticmethod
+    def _is_false_not_exists(e: CogniteAPIError, request_views: set[ViewId]) -> bool:
+        """This is a bug in the API where it returns a 400 status complaining that a views does not exist,
+        even though they are part of the request. This bug is reported to Cognite. The workaround is to do a
+        topological sort of the strongly connected views and retry the request.
+        """
+        if "not exist" not in e.message and 400 <= e.code < 500:
+            return False
+        results = re.findall(r"'([a-zA-Z0-9_-]+):([a-zA-Z0-9_]+)/([.a-zA-Z0-9_-]+)'", e.message)
+        if not results:
+            # No view references in the message
+            return False
+        error_message_views = {ViewId(*result) for result in results}
+        return error_message_views.issubset(request_views)
+
     @overload
     def retrieve(
         self,

cognite/neat/core/_data_model/models/physical/_exporter.py

@@ -391,9 +391,9 @@ class _DMSExporter:
                 container.indexes = container.indexes or {}
                 index_property_list = [prop_id for prop_id, _ in sorted(properties, key=lambda x: x[1].order or 0)]
                 index_entity = properties[0][1]
-                if index_entity.prefix == "inverted" or (index_entity.prefix is Undefined and is_list):
+                if index_entity.prefix == "inverted" or (index_entity.prefix == Undefined and is_list):
                     container.indexes[index_name] = InvertedIndex(properties=index_property_list)
-                elif index_entity.prefix == "btree" or (index_entity.prefix is Undefined and not is_list):
+                elif index_entity.prefix == "btree" or (index_entity.prefix == Undefined and not is_list):
                     container.indexes[index_name] = BTreeIndex(
                         properties=index_property_list, cursorable=index_entity.cursorable or False
                     )

cognite/neat/core/_utils/tarjan.py

@@ -0,0 +1,44 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def tarjan(dependencies_by_id: dict[T, set[T]]) -> list[set[T]]:
+    """Returns the strongly connected components of the dependency graph
+     in topological order.
+    Args:
+        dependencies_by_id: A dictionary where the keys are ids and the values are sets of ids that the key depends on.
+    Returns:
+        A list of sets of ids that are strongly connected components in the dependency graph.
+    """
+
+    stack = []
+    stack_set = set()
+    index: dict[T, int] = {}
+    lowlink = {}
+    result = []
+
+    def visit(v: T) -> None:
+        index[v] = len(index)
+        lowlink[v] = index[v]
+        stack.append(v)
+        stack_set.add(v)
+        for w in dependencies_by_id.get(v, []):
+            if w not in index:
+                visit(w)
+                lowlink[v] = min(lowlink[w], lowlink[v])
+            elif w in stack_set:
+                lowlink[v] = min(lowlink[v], index[w])
+        if lowlink[v] == index[v]:
+            scc = set()
+            dependency: T | None = None
+            while v != dependency:
+                dependency = stack.pop()
+                scc.add(dependency)
+                stack_set.remove(dependency)
+            result.append(scc)
+
+    for view_id in dependencies_by_id.keys():
+        if view_id not in index:
+            visit(view_id)
+    return result

{cognite_neat-0.123.6.dist-info → cognite_neat-0.123.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.123.6
+Version: 0.123.8
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

{cognite_neat-0.123.6.dist-info → cognite_neat-0.123.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=12StS1dzH9_MElqxGvLWrNsxCJl9Hv8A2a9D0E5OD_U,193
-cognite/neat/_version.py,sha256=pYgIOIoZmd2M8dmiRwHzXC1Amc9xjpZBXaTTBA_r8-Q,46
+cognite/neat/_version.py,sha256=p4Io2TCN-JUEOMcJwjUduCAyqkMMmElUmdDGzpevtQU,46
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/core/_config.py,sha256=WT1BS8uADcFvGoUYOOfwFOVq_VBl472TisdoA3wLick,280
@@ -9,7 +9,7 @@ cognite/neat/core/_client/__init__.py,sha256=RQ7MwL8mwGqGHokRzsPqO3XStDzmI4-c12_
 cognite/neat/core/_client/_api_client.py,sha256=CqgG4kEArI9jiKAh82YrRZv_SzeMKA5guIZOvDg2R5I,860
 cognite/neat/core/_client/testing.py,sha256=rZGd-TFwNtfUqT8LV0u3FT0kHwNrjnvDNZU_Mcd5yx4,1329
 cognite/neat/core/_client/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/neat/core/_client/_api/data_modeling_loaders.py,sha256=YEZP0V_RXenzzFSxDzt4SbVnpzB4VP0fcI-Bab1VLw8,39815
+cognite/neat/core/_client/_api/data_modeling_loaders.py,sha256=6ZiL_btoqMlJF2TtW9q2y8zEWds7QWLn4YlFH72v1Ew,43794
 cognite/neat/core/_client/_api/neat_instances.py,sha256=8HcQO1sp8zjXdnRBRQ4yeQzt1O906HNSrDDCgrgTe8A,4805
 cognite/neat/core/_client/_api/schema.py,sha256=lbA8Cka_7K_RjmaxdeqkVkIwKPfWeDvpYvvEOGI07xo,6967
 cognite/neat/core/_client/_api/statistics.py,sha256=M0JpCHD6WMfggoe-HyXfeigwRCvZJjVF-xNB9CgB4UE,3796
@@ -65,7 +65,7 @@ cognite/neat/core/_data_model/models/mapping/__init__.py,sha256=T68Hf7rhiXa7b03h
 cognite/neat/core/_data_model/models/mapping/_classic2core.py,sha256=FRDpYP_CX-CfptdFCZmfqsbKCYQ9BQPUbKoifTICe30,1415
 cognite/neat/core/_data_model/models/mapping/_classic2core.yaml,sha256=ei-nuivNWVW9HmvzDBKIPF6ZdgaMq64XHw_rKm0CMxg,22584
 cognite/neat/core/_data_model/models/physical/__init__.py,sha256=ONE_xLw1cxfw88rICG_RtbjCYUZm8yS2kBQ4Di3EGnA,987
-cognite/neat/core/_data_model/models/physical/_exporter.py,sha256=WHYjWyiKpA7GWO2_vPN3AphpDr8OjzxI5JMyO7b3sYQ,30083
+cognite/neat/core/_data_model/models/physical/_exporter.py,sha256=DPOytV-sIzpGJtfDEfi7G4RWnSCVNRLWe1KzY26ewmc,30083
 cognite/neat/core/_data_model/models/physical/_unverified.py,sha256=VyI-JULAu6kHJygUclDPH1JYjhf_XcO58tI9BkXORC0,18430
 cognite/neat/core/_data_model/models/physical/_validation.py,sha256=uPzbcJmFj0g9ATVq86rUsmSGOfQ0iBeIevoKh1lCu98,33179
 cognite/neat/core/_data_model/models/physical/_verified.py,sha256=Fvy0pOYk03nHDlRz_SECg9ALW3y0LaCZG6XNBduCi9Q,24332
@@ -148,6 +148,7 @@ cognite/neat/core/_utils/graph_transformations_report.py,sha256=ORVH7lw357TPOq4e
 cognite/neat/core/_utils/io_.py,sha256=D2Mg8sOxfBoDg3fC0jBzaxO3vkXmr0QvZSgYIv6xRkM,386
 cognite/neat/core/_utils/rdf_.py,sha256=8AALp8H_nXEDSBo6jZ1idyT_x3K4PJT5ZyBEyxPmgxI,10403
 cognite/neat/core/_utils/spreadsheet.py,sha256=MMI_1zxeHEf9Ggu_-t_ryjj54ky085QIf8eArt5hXEY,5749
+cognite/neat/core/_utils/tarjan.py,sha256=IZvwaIITryGVNbo9Bv5EA9_sW3DyfUNAe7uYyPOCL0g,1357
 cognite/neat/core/_utils/text.py,sha256=ON4ihfscFJkQqQ-Rj46XXtf-9tAobwXbbfa3wuekSu4,8519
 cognite/neat/core/_utils/time_.py,sha256=7ayUm0OWZm1JDmy32E4ip8WRr2o0GLwrHwJA8sJ43Z4,357
 cognite/neat/core/_utils/upload.py,sha256=yR-BvvrWPh0XHoIGByXMEVi3JONzmc5xwXbmEDBdA8U,5860
@@ -185,7 +186,7 @@ cognite/neat/session/engine/__init__.py,sha256=D3MxUorEs6-NtgoICqtZ8PISQrjrr4dvc
 cognite/neat/session/engine/_import.py,sha256=1QxA2_EK613lXYAHKQbZyw2yjo5P9XuiX4Z6_6-WMNQ,169
 cognite/neat/session/engine/_interface.py,sha256=3W-cYr493c_mW3P5O6MKN1xEQg3cA7NHR_ev3zdF9Vk,533
 cognite/neat/session/engine/_load.py,sha256=g52uYakQM03VqHt_RDHtpHso1-mFFifH5M4T2ScuH8A,5198
-cognite_neat-0.123.6.dist-info/METADATA,sha256=ejxcaEkDxrf0agU1343EbR4eDXUqUwwgku3eEJ90TvE,9171
-cognite_neat-0.123.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_neat-0.123.6.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.123.6.dist-info/RECORD,,
+cognite_neat-0.123.8.dist-info/METADATA,sha256=SxWb1954YW95XT55Z_I9Z9BULuePbOcYuWg8GhfdHLY,9171
+cognite_neat-0.123.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_neat-0.123.8.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.123.8.dist-info/RECORD,,