cognite-neat 0.123.5__py3-none-any.whl → 0.123.7__py3-none-any.whl

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.123.5"
+__version__ = "0.123.7"
 __engine__ = "^2.0.4"

cognite/neat/core/_client/_api/data_modeling_loaders.py

@@ -1,8 +1,10 @@
+import re
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from collections.abc import Callable, Collection, Iterable, Sequence
 from dataclasses import dataclass, field
-from graphlib import TopologicalSorter
+from graphlib import CycleError, TopologicalSorter
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeVar, cast, overload
 
 from cognite.client.data_classes import filters
@@ -56,6 +58,7 @@ from cognite.neat.core._client.data_classes.data_modeling import Component
 from cognite.neat.core._client.data_classes.schema import DMSSchema
 from cognite.neat.core._issues.warnings import CDFMaxIterationsWarning
 from cognite.neat.core._shared import T_ID
+from cognite.neat.core._utils.tarjan import tarjan
 
 if TYPE_CHECKING:
     from cognite.neat.core._client._api_client import NeatClient
@@ -620,6 +623,81 @@ class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, View
     def _create(self, items: Sequence[ViewApply]) -> ViewList:
         return self._client.data_modeling.views.apply(items)
 
+    def create(self, items: Sequence[ViewApply]) -> ViewList:
+        try:
+            return self._create(items)
+        except CogniteAPIError as e1:
+            if self._is_auto_retryable(e1):
+                # Fallback to creating one by one if the error is auto-retryable.
+                return self._fallback_one_by_one(self._create, items)
+            elif self._is_false_not_exists(e1, {item.as_id() for item in items}):
+                return self._try_to_recover_coupled(items, e1)
+            raise
+
+    def _try_to_recover_coupled(self, items: Sequence[ViewApply], original_error: CogniteAPIError) -> ViewList:
+        """The /models/views endpoint can give faulty 400 about missing views that are part of the request.
+
+        This method tries to recover from such errors by identifying the strongly connected components in the graph
+        defined by the implements and through properties of the views. We then create the components in topological
+        order.
+
+        Args:
+            items: The items that failed to create.
+            original_error: The original error that was raised. If the problem is not recoverable, this error is raised.
+
+        Returns:
+            The views that were created.
+
+        """
+        views_by_id = {self.get_id(item): item for item in items}
+        parents_by_child: dict[ViewId, set[ViewId]] = {
+            view_id: {parent for parent in view.implements or [] if parent in views_by_id}
+            for view_id, view in views_by_id.items()
+        }
+        # Check for cycles in the implements graph
+        try:
+            TopologicalSorter(parents_by_child).static_order()
+        except CycleError as e:
+            raise CycleError(f"Failed to deploy views. This likely due to a cycle in implements. {e.args[1]}") from None
+
+        dependencies_by_id: dict[ViewId, set[ViewId]] = defaultdict(set)
+        for view_id, view in views_by_id.items():
+            dependencies_by_id[view_id].update([parent for parent in view.implements or [] if parent in views_by_id])
+            for properties in (view.properties or {}).values():
+                if isinstance(properties, ReverseDirectRelationApply):
+                    if isinstance(properties.through.source, ViewId) and properties.through.source in views_by_id:
+                        dependencies_by_id[view_id].add(properties.through.source)
+
+        created = ViewList([])
+        for strongly_connected in tarjan(dependencies_by_id):
+            to_create = [views_by_id[view_id] for view_id in strongly_connected]
+            try:
+                created_set = self._client.data_modeling.views.apply(to_create)
+            except CogniteAPIError:
+                self._client.data_modeling.views.delete(created.as_ids())
+                raise original_error from None
+            created.extend(created_set)
+        return created
+
+    @staticmethod
+    def _is_auto_retryable(e: CogniteAPIError) -> bool:
+        return isinstance(e.extra, dict) and "isAutoRetryable" in e.extra and e.extra["isAutoRetryable"]
+
+    @staticmethod
+    def _is_false_not_exists(e: CogniteAPIError, request_views: set[ViewId]) -> bool:
+        """This is a bug in the API where it returns a 400 status complaining that a views does not exist,
+        even though they are part of the request. This bug is reported to Cognite. The workaround is to do a
+        topological sort of the strongly connected views and retry the request.
+        """
+        if "not exist" not in e.message and 400 <= e.code < 500:
+            return False
+        results = re.findall(r"'([a-zA-Z0-9_-]+):([a-zA-Z0-9_]+)/([.a-zA-Z0-9_-]+)'", e.message)
+        if not results:
+            # No view references in the message
+            return False
+        error_message_views = {ViewId(*result) for result in results}
+        return error_message_views.issubset(request_views)
+
     @overload
     def retrieve(
         self,

cognite/neat/core/_store/_data_model.py

@@ -355,6 +355,7 @@ class NeatDataModelStore:
 
         agent = exporter.agent
         start = datetime.now(timezone.utc)
+        result: UploadResultList | Path | URIRef | None = None
         with catch_issues() as issue_list:
             # Validate the type of the result
             result = action(input_, *exporter_args)

cognite/neat/core/_utils/tarjan.py

@@ -0,0 +1,44 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def tarjan(dependencies_by_id: dict[T, set[T]]) -> list[set[T]]:
+    """Returns the strongly connected components of the dependency graph
+     in topological order.
+    Args:
+        dependencies_by_id: A dictionary where the keys are ids and the values are sets of ids that the key depends on.
+    Returns:
+        A list of sets of ids that are strongly connected components in the dependency graph.
+    """
+
+    stack = []
+    stack_set = set()
+    index: dict[T, int] = {}
+    lowlink = {}
+    result = []
+
+    def visit(v: T) -> None:
+        index[v] = len(index)
+        lowlink[v] = index[v]
+        stack.append(v)
+        stack_set.add(v)
+        for w in dependencies_by_id.get(v, []):
+            if w not in index:
+                visit(w)
+                lowlink[v] = min(lowlink[w], lowlink[v])
+            elif w in stack_set:
+                lowlink[v] = min(lowlink[v], index[w])
+        if lowlink[v] == index[v]:
+            scc = set()
+            dependency: T | None = None
+            while v != dependency:
+                dependency = stack.pop()
+                scc.add(dependency)
+                stack_set.remove(dependency)
+            result.append(scc)
+
+    for view_id in dependencies_by_id.keys():
+        if view_id not in index:
+            visit(view_id)
+    return result

{cognite_neat-0.123.5.dist-info → cognite_neat-0.123.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.123.5
+Version: 0.123.7
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

{cognite_neat-0.123.5.dist-info → cognite_neat-0.123.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cognite/neat/__init__.py,sha256=12StS1dzH9_MElqxGvLWrNsxCJl9Hv8A2a9D0E5OD_U,193
-cognite/neat/_version.py,sha256=CGPu04zqANFzgwV9cLTmb2E85DRuSaBx8spD-wNxD-4,46
+cognite/neat/_version.py,sha256=qkkjBeBYYbUi5FDXyPZvuIZ1C3jviVckqO158UiM9mA,46
 cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/neat/core/_config.py,sha256=WT1BS8uADcFvGoUYOOfwFOVq_VBl472TisdoA3wLick,280
@@ -9,7 +9,7 @@ cognite/neat/core/_client/__init__.py,sha256=RQ7MwL8mwGqGHokRzsPqO3XStDzmI4-c12_
 cognite/neat/core/_client/_api_client.py,sha256=CqgG4kEArI9jiKAh82YrRZv_SzeMKA5guIZOvDg2R5I,860
 cognite/neat/core/_client/testing.py,sha256=rZGd-TFwNtfUqT8LV0u3FT0kHwNrjnvDNZU_Mcd5yx4,1329
 cognite/neat/core/_client/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/neat/core/_client/_api/data_modeling_loaders.py,sha256=YEZP0V_RXenzzFSxDzt4SbVnpzB4VP0fcI-Bab1VLw8,39815
+cognite/neat/core/_client/_api/data_modeling_loaders.py,sha256=6ZiL_btoqMlJF2TtW9q2y8zEWds7QWLn4YlFH72v1Ew,43794
 cognite/neat/core/_client/_api/neat_instances.py,sha256=8HcQO1sp8zjXdnRBRQ4yeQzt1O906HNSrDDCgrgTe8A,4805
 cognite/neat/core/_client/_api/schema.py,sha256=lbA8Cka_7K_RjmaxdeqkVkIwKPfWeDvpYvvEOGI07xo,6967
 cognite/neat/core/_client/_api/statistics.py,sha256=M0JpCHD6WMfggoe-HyXfeigwRCvZJjVF-xNB9CgB4UE,3796
@@ -136,7 +136,7 @@ cognite/neat/core/_issues/warnings/_properties.py,sha256=h7jGO6wfjAd1-T9u-BByUf7
 cognite/neat/core/_issues/warnings/_resources.py,sha256=_iPRq0pRMmRu3LFjqZTaG3OqOzw4f8-Vc9G4Im__FHc,3578
 cognite/neat/core/_issues/warnings/user_modeling.py,sha256=Qn_S8TLw7MMYQaJcZBScJA48kz_PrTWz0NaepSR70Fk,4144
 cognite/neat/core/_store/__init__.py,sha256=wpsF8xjIQ5V21NOh45XQV813n_EzgyPOt0VVinYjnDI,140
-cognite/neat/core/_store/_data_model.py,sha256=a7061nfqmiWConIU0CBDISEjoY_dwHLzvA3PHk8-KRY,19583
+cognite/neat/core/_store/_data_model.py,sha256=6Eg_QH1ZlS4AO3c0_Ye8M-Y4_Ycf3mQPJBKyardGu9w,19646
 cognite/neat/core/_store/_instance.py,sha256=QFrhOyaVA-mOtRaKZDE0aYyEJlSYx1jImKC8iE3b75E,17406
 cognite/neat/core/_store/_provenance.py,sha256=aMEsq27dZ4NZ6XEC8hA0fIDF13i1ZP3QwnclLCRaNk4,7326
 cognite/neat/core/_store/exceptions.py,sha256=jcd1Gv65mfTdC4cipFAMWUNghEmdLS_lwPH1FB_ebxI,1656
@@ -148,6 +148,7 @@ cognite/neat/core/_utils/graph_transformations_report.py,sha256=ORVH7lw357TPOq4e
 cognite/neat/core/_utils/io_.py,sha256=D2Mg8sOxfBoDg3fC0jBzaxO3vkXmr0QvZSgYIv6xRkM,386
 cognite/neat/core/_utils/rdf_.py,sha256=8AALp8H_nXEDSBo6jZ1idyT_x3K4PJT5ZyBEyxPmgxI,10403
 cognite/neat/core/_utils/spreadsheet.py,sha256=MMI_1zxeHEf9Ggu_-t_ryjj54ky085QIf8eArt5hXEY,5749
+cognite/neat/core/_utils/tarjan.py,sha256=IZvwaIITryGVNbo9Bv5EA9_sW3DyfUNAe7uYyPOCL0g,1357
 cognite/neat/core/_utils/text.py,sha256=ON4ihfscFJkQqQ-Rj46XXtf-9tAobwXbbfa3wuekSu4,8519
 cognite/neat/core/_utils/time_.py,sha256=7ayUm0OWZm1JDmy32E4ip8WRr2o0GLwrHwJA8sJ43Z4,357
 cognite/neat/core/_utils/upload.py,sha256=yR-BvvrWPh0XHoIGByXMEVi3JONzmc5xwXbmEDBdA8U,5860
@@ -185,7 +186,7 @@ cognite/neat/session/engine/__init__.py,sha256=D3MxUorEs6-NtgoICqtZ8PISQrjrr4dvc
 cognite/neat/session/engine/_import.py,sha256=1QxA2_EK613lXYAHKQbZyw2yjo5P9XuiX4Z6_6-WMNQ,169
 cognite/neat/session/engine/_interface.py,sha256=3W-cYr493c_mW3P5O6MKN1xEQg3cA7NHR_ev3zdF9Vk,533
 cognite/neat/session/engine/_load.py,sha256=g52uYakQM03VqHt_RDHtpHso1-mFFifH5M4T2ScuH8A,5198
-cognite_neat-0.123.5.dist-info/METADATA,sha256=8hvMf17cPBhpa0siFt9Ji0ARsA91KiKz0bRNAj6t8BY,9171
-cognite_neat-0.123.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_neat-0.123.5.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
-cognite_neat-0.123.5.dist-info/RECORD,,
+cognite_neat-0.123.7.dist-info/METADATA,sha256=qLpEiC8HWa4kFXIfty5_vNMFWfJ89CybIzFVA0FMxD0,9171
+cognite_neat-0.123.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_neat-0.123.7.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
+cognite_neat-0.123.7.dist-info/RECORD,,