cognite-neat 0.123.6__tar.gz → 0.123.8__tar.gz

{cognite_neat-0.123.6 → cognite_neat-0.123.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.123.6
+Version: 0.123.8
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

cognite_neat-0.123.8/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.123.8"
+__engine__ = "^2.0.4"

{cognite_neat-0.123.6 → cognite_neat-0.123.8}/cognite/neat/core/_client/_api/data_modeling_loaders.py

@@ -1,8 +1,10 @@
+import re
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from collections.abc import Callable, Collection, Iterable, Sequence
 from dataclasses import dataclass, field
-from graphlib import TopologicalSorter
+from graphlib import CycleError, TopologicalSorter
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeVar, cast, overload
 
 from cognite.client.data_classes import filters
@@ -56,6 +58,7 @@ from cognite.neat.core._client.data_classes.data_modeling import Component
 from cognite.neat.core._client.data_classes.schema import DMSSchema
 from cognite.neat.core._issues.warnings import CDFMaxIterationsWarning
 from cognite.neat.core._shared import T_ID
+from cognite.neat.core._utils.tarjan import tarjan
 
 if TYPE_CHECKING:
     from cognite.neat.core._client._api_client import NeatClient
@@ -620,6 +623,81 @@ class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, View
     def _create(self, items: Sequence[ViewApply]) -> ViewList:
         return self._client.data_modeling.views.apply(items)
 
+    def create(self, items: Sequence[ViewApply]) -> ViewList:
+        try:
+            return self._create(items)
+        except CogniteAPIError as e1:
+            if self._is_auto_retryable(e1):
+                # Fallback to creating one by one if the error is auto-retryable.
+                return self._fallback_one_by_one(self._create, items)
+            elif self._is_false_not_exists(e1, {item.as_id() for item in items}):
+                return self._try_to_recover_coupled(items, e1)
+            raise
+
+    def _try_to_recover_coupled(self, items: Sequence[ViewApply], original_error: CogniteAPIError) -> ViewList:
+        """The /models/views endpoint can give faulty 400 about missing views that are part of the request.
+
+        This method tries to recover from such errors by identifying the strongly connected components in the graph
+        defined by the implements and through properties of the views. We then create the components in topological
+        order.
+
+        Args:
+            items: The items that failed to create.
+            original_error: The original error that was raised. If the problem is not recoverable, this error is raised.
+
+        Returns:
+            The views that were created.
+
+        """
+        views_by_id = {self.get_id(item): item for item in items}
+        parents_by_child: dict[ViewId, set[ViewId]] = {
+            view_id: {parent for parent in view.implements or [] if parent in views_by_id}
+            for view_id, view in views_by_id.items()
+        }
+        # Check for cycles in the implements graph
+        try:
+            TopologicalSorter(parents_by_child).static_order()
+        except CycleError as e:
+            raise CycleError(f"Failed to deploy views. This likely due to a cycle in implements. {e.args[1]}") from None
+
+        dependencies_by_id: dict[ViewId, set[ViewId]] = defaultdict(set)
+        for view_id, view in views_by_id.items():
+            dependencies_by_id[view_id].update([parent for parent in view.implements or [] if parent in views_by_id])
+            for properties in (view.properties or {}).values():
+                if isinstance(properties, ReverseDirectRelationApply):
+                    if isinstance(properties.through.source, ViewId) and properties.through.source in views_by_id:
+                        dependencies_by_id[view_id].add(properties.through.source)
+
+        created = ViewList([])
+        for strongly_connected in tarjan(dependencies_by_id):
+            to_create = [views_by_id[view_id] for view_id in strongly_connected]
+            try:
+                created_set = self._client.data_modeling.views.apply(to_create)
+            except CogniteAPIError:
+                self._client.data_modeling.views.delete(created.as_ids())
+                raise original_error from None
+            created.extend(created_set)
+        return created
+
+    @staticmethod
+    def _is_auto_retryable(e: CogniteAPIError) -> bool:
+        return isinstance(e.extra, dict) and "isAutoRetryable" in e.extra and e.extra["isAutoRetryable"]
+
+    @staticmethod
+    def _is_false_not_exists(e: CogniteAPIError, request_views: set[ViewId]) -> bool:
+        """This is a bug in the API where it returns a 400 status complaining that a views does not exist,
+        even though they are part of the request. This bug is reported to Cognite. The workaround is to do a
+        topological sort of the strongly connected views and retry the request.
+        """
+        if "not exist" not in e.message and 400 <= e.code < 500:
+            return False
+        results = re.findall(r"'([a-zA-Z0-9_-]+):([a-zA-Z0-9_]+)/([.a-zA-Z0-9_-]+)'", e.message)
+        if not results:
+            # No view references in the message
+            return False
+        error_message_views = {ViewId(*result) for result in results}
+        return error_message_views.issubset(request_views)
+
     @overload
     def retrieve(
         self,

{cognite_neat-0.123.6 → cognite_neat-0.123.8}/cognite/neat/core/_data_model/models/physical/_exporter.py

@@ -391,9 +391,9 @@ class _DMSExporter:
                 container.indexes = container.indexes or {}
                 index_property_list = [prop_id for prop_id, _ in sorted(properties, key=lambda x: x[1].order or 0)]
                 index_entity = properties[0][1]
-                if index_entity.prefix == "inverted" or (index_entity.prefix is Undefined and is_list):
+                if index_entity.prefix == "inverted" or (index_entity.prefix == Undefined and is_list):
                     container.indexes[index_name] = InvertedIndex(properties=index_property_list)
-                elif index_entity.prefix == "btree" or (index_entity.prefix is Undefined and not is_list):
+                elif index_entity.prefix == "btree" or (index_entity.prefix == Undefined and not is_list):
                     container.indexes[index_name] = BTreeIndex(
                         properties=index_property_list, cursorable=index_entity.cursorable or False
                     )

cognite_neat-0.123.8/cognite/neat/core/_utils/tarjan.py

@@ -0,0 +1,44 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def tarjan(dependencies_by_id: dict[T, set[T]]) -> list[set[T]]:
+    """Returns the strongly connected components of the dependency graph
+     in topological order.
+    Args:
+        dependencies_by_id: A dictionary where the keys are ids and the values are sets of ids that the key depends on.
+    Returns:
+        A list of sets of ids that are strongly connected components in the dependency graph.
+    """
+
+    stack = []
+    stack_set = set()
+    index: dict[T, int] = {}
+    lowlink = {}
+    result = []
+
+    def visit(v: T) -> None:
+        index[v] = len(index)
+        lowlink[v] = index[v]
+        stack.append(v)
+        stack_set.add(v)
+        for w in dependencies_by_id.get(v, []):
+            if w not in index:
+                visit(w)
+                lowlink[v] = min(lowlink[w], lowlink[v])
+            elif w in stack_set:
+                lowlink[v] = min(lowlink[v], index[w])
+        if lowlink[v] == index[v]:
+            scc = set()
+            dependency: T | None = None
+            while v != dependency:
+                dependency = stack.pop()
+                scc.add(dependency)
+                stack_set.remove(dependency)
+            result.append(scc)
+
+    for view_id in dependencies_by_id.keys():
+        if view_id not in index:
+            visit(view_id)
+    return result

cognite_neat-0.123.8/last_changelog_entry.md

@@ -0,0 +1,6 @@
+
+### Fixed
+
+- Neat no longer silently fails for `neat.to.cdf.data_model(...)` when
+indices does not specify `btree` or `inverted`. Note this bug was
+introduced in `0.123.0`

cognite_neat-0.123.8/last_git_message.txt

@@ -0,0 +1,25 @@
+[NEAT-1122] 🐛 Inferred indices (#1243)
+
+# Description
+
+In the conversion of index from phycial model to DMS schema, I assumed
+in the original implementation that `Undefined` is a singleton and it
+turns out that is not the case.
+
+The following screenshot is from stopping the conversion of indices:
+
+![image](https://github.com/user-attachments/assets/fb16396d-2c06-4093-8fde-e80705ca4570)
+
+
+## Bump
+
+- [x] Patch
+- [ ] Minor
+- [ ] Skip
+
+## Changelog
+### Fixed
+
+- Neat no longer silently fails for `neat.to.cdf.data_model(...)` when
+indices does not specify `btree` or `inverted`. Note this bug was
+introduced in `0.123.0`

cognite_neat-0.123.8/last_version.txt

@@ -0,0 +1 @@
+0.123.7

{cognite_neat-0.123.6 → cognite_neat-0.123.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-neat"
-version = "0.123.6"
+version = "0.123.8"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [
@@ -94,6 +94,7 @@ dev = [
     "cognite-toolkit >= 0.4,< 0.5",
     "tqdm>=4.67.1, <5",
     "deepdiff>=8.2.0, <9.0.0",
+    "hypothesis>=6.135.0, <7.0.0",
 ]
 
 [tool.uv]

{cognite_neat-0.123.6 → cognite_neat-0.123.8}/tests/data/__init__.py

@@ -70,10 +70,12 @@ class SchemaData:
         dms_unknown_value_type_xlsx = _physical / "dms-unknown-value-type.xlsx"
         isa_plus_cdm_xlsx = _physical / "isa_plus_cdm.xlsx"
         missing_in_model_value_xlsx = _physical / "missing-in-model-value.xlsx"
+        physical_singleton_issue_xlsx = _physical / "physical_singleton_issue.xlsx"
         mixed_up_version_xlsx = _physical / "mixed-up-version.xlsx"
         pump_example_duplicated_resources_xlsx = _physical / "pump_example_duplicated_resources.xlsx"
         pump_example_with_missing_cells_xlsx = _physical / "pump_example_with_missing_cells.xlsx"
         pump_example_with_missing_cells_raise_issues = _physical / "pump_example_with_missing_cells_raise_issues.xlsx"
+        strongly_connected_model_folder = _physical / "strongly_connected_model"
 
     class Conversion:
         conversion = _schema_dir / "conversion"

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/1.Production.container.yaml

@@ -0,0 +1,38 @@
+space: strongly-coupled-model
+externalId: Production
+name: Production
+description: Missing
+properties:
+  id:
+    type:
+      list: false
+      collation: ucs_basic
+      type: text
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: id
+    description: Missing
+  type:
+    type:
+      list: false
+      collation: ucs_basic
+      type: text
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: type
+constraints:
+  cogniteAssetPresent:
+    require:
+      space: cdf_cdm
+      externalId: CogniteAsset
+      type: container
+    constraintType: requires
+indexes:
+  id:
+    properties:
+    - id
+    cursorable: true
+    indexType: btree
+usedFor: node

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/10.SimNode.container.yaml

@@ -0,0 +1,22 @@
+space: strongly-coupled-model
+externalId: SimNode
+name: SimNode
+usedFor: node
+description: Missing
+properties:
+  coordinate:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: coordinate
+constraints:
+  cogniteDescribablePresent:
+    require:
+      space: strongly-coupled-model
+      externalId: SimNodeAndEdge
+      type: container
+    constraintType: requires
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/12.SimSubNode.container.yaml

@@ -0,0 +1,22 @@
+space: strongly-coupled-model
+externalId: SimSubNode
+name: SimSubNode
+usedFor: node
+description: Missing
+properties:
+  parent:
+    type:
+      list: false
+      type: direct
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: parent
+constraints:
+  cogniteDescribablePresent:
+    require:
+      space: strongly-coupled-model
+      externalId: SimNode
+      type: container
+    constraintType: requires
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/13.SimEdge.container.yaml

@@ -0,0 +1,38 @@
+space: strongly-coupled-model
+externalId: SimEdge
+name: SimEdge
+usedFor: node
+description: Missing
+properties:
+  source:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: source
+  destination:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: destination
+  coordinates:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: coordinates
+constraints:
+  cogniteDescribablePresent:
+    require:
+      space: strongly-coupled-model
+      externalId: SimNodeAndEdge
+      type: container
+    constraintType: requires
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/14.Property.container.yaml

@@ -0,0 +1,67 @@
+space: strongly-coupled-model
+externalId: Property
+name: Property
+usedFor: node
+description: Missing
+properties:
+  type:
+    type:
+      type: enum
+      values:
+        Input: {}
+        Output: {}
+    immutable: true
+    nullable: false
+    autoIncrement: false
+    name: type
+    description: Missing
+  nodeOrEdge:
+    type:
+      list: false
+      type: direct
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: nodeOrEdge
+  workflow:
+    type:
+      type: text
+      list: false
+      collation: ucs_basic
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: workflow
+  unit:
+    type:
+      type: text
+      list: false
+      collation: ucs_basic
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: unit
+  valueClassicRef:
+    type:
+      type: timeseries
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: value
+  valueRef:
+    type:
+      list: false
+      type: direct
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: valueRef
+constraints:
+  cogniteDescribablePresent:
+    require:
+      space: cdf_cdm
+      externalId: CogniteDescribable
+      type: container
+    constraintType: requires
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/15.Coordinate.container.yaml

@@ -0,0 +1,41 @@
+space: strongly-coupled-model
+externalId: Coordinate
+name: Coordinate
+usedFor: node
+description: Missing
+properties:
+  elevation:
+    type:
+      type: float64
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: elevation
+  lat:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: lat
+  long:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: long
+  crs:
+    type:
+      type: text
+      list: true
+      collation: ucs_basic
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: crs
+constraints: {}
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/16.Coordinates.container.yaml

@@ -0,0 +1,49 @@
+space: strongly-coupled-model
+externalId: Coordinates
+name: Coordinates
+usedFor: node
+description: Missing
+properties:
+  elevation:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: elevation
+  lat:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: lat
+  long:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: long
+  horizontalDistance:
+    type:
+      type: float64
+      list: true
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: horizontalDistance
+  crs:
+    type:
+      type: text
+      list: true
+      collation: ucs_basic
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: crs
+constraints: {}
+indexes: {}

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/17.ModelExecutionEnvironment.view.yaml

@@ -0,0 +1,25 @@
+externalId: ModelExecutionEnvironment
+space: strongly-coupled-model
+version: v1
+name: ModelExecutionEnvironment
+description: Missing
+properties:
+  type:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: ModelExecutionEnvironment
+    containerPropertyIdentifier: type
+    name: type
+  version:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: ModelExecutionEnvironment
+    containerPropertyIdentifier: version
+    name: version
+implements:
+- type: view
+  space: cdf_cdm
+  externalId: CogniteDescribable
+  version: v1

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/18.Coordinate.view.yaml

@@ -0,0 +1,35 @@
+externalId: Coordinate
+space: strongly-coupled-model
+version: v1
+name: Coordinate
+description: Missing
+properties:
+  elevation:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinate
+    containerPropertyIdentifier: elevation
+    name: elevation
+  lat:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinate
+    containerPropertyIdentifier: lat
+    name: lat
+  long:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinate
+    containerPropertyIdentifier: long
+    name: long
+  crs:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinate
+    containerPropertyIdentifier: crs
+    name: crs
+implements: []

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/19.Coordinates.view.yaml

@@ -0,0 +1,42 @@
+externalId: Coordinates
+space: strongly-coupled-model
+version: v1
+name: Coordinates
+description: Missing
+properties:
+  elevation:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinates
+    containerPropertyIdentifier: elevation
+    name: elevation
+  lat:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinates
+    containerPropertyIdentifier: lat
+    name: lat
+  long:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinates
+    containerPropertyIdentifier: long
+    name: long
+  horizontalDistance:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinates
+    containerPropertyIdentifier: horizontalDistance
+    name: horizontalDistance
+  crs:
+    container:
+      type: container
+      space: strongly-coupled-model
+      externalId: Coordinates
+    containerPropertyIdentifier: crs
+    name: crs
+implements: []

cognite_neat-0.123.8/tests/data/_schema/physical/strongly_connected_model/data_models/2.SimulationModel.container.yaml

@@ -0,0 +1,44 @@
+space: strongly-coupled-model
+externalId: SimulationModel
+name: SimulationModel
+usedFor: node
+description: Missing
+properties:
+  modelId:
+    type:
+      type: text
+      list: false
+      collation: ucs_basic
+    immutable: false
+    nullable: false
+    autoIncrement: false
+    name: modelId
+  modelExecutionEnvironment:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: modelExecutionEnvironment
+  entity:
+    type:
+      type: direct
+      list: false
+    immutable: false
+    nullable: true
+    autoIncrement: false
+    name: entity
+constraints:
+  cogniteDescribablePresent:
+    require:
+      space: cdf_cdm
+      externalId: CogniteDescribable
+      type: container
+    constraintType: requires
+indexes:
+  modelId:
+    properties:
+    - modelId
+    cursorable: true
+    indexType: btree