cognite-neat 0.100.1__py3-none-any.whl → 0.102.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (32) hide show
  1. cognite/neat/_constants.py +5 -1
  2. cognite/neat/_graph/loaders/_rdf2dms.py +1 -2
  3. cognite/neat/_graph/queries/_base.py +22 -2
  4. cognite/neat/_graph/queries/_shared.py +4 -4
  5. cognite/neat/_graph/transformers/__init__.py +17 -0
  6. cognite/neat/_graph/transformers/_base.py +1 -1
  7. cognite/neat/_graph/transformers/_iodd.py +9 -4
  8. cognite/neat/_graph/transformers/_prune_graph.py +196 -65
  9. cognite/neat/_rules/exporters/_rules2dms.py +35 -13
  10. cognite/neat/_rules/exporters/_rules2excel.py +7 -2
  11. cognite/neat/_rules/importers/_dms2rules.py +51 -19
  12. cognite/neat/_rules/importers/_rdf/_base.py +2 -2
  13. cognite/neat/_rules/models/_base_rules.py +13 -9
  14. cognite/neat/_rules/models/dms/_rules.py +111 -39
  15. cognite/neat/_rules/models/information/_rules.py +52 -19
  16. cognite/neat/_session/_base.py +18 -0
  17. cognite/neat/_session/_prepare.py +85 -2
  18. cognite/neat/_session/_read.py +3 -3
  19. cognite/neat/_session/_to.py +1 -1
  20. cognite/neat/_session/engine/_load.py +3 -1
  21. cognite/neat/_store/_base.py +23 -2
  22. cognite/neat/_utils/auth.py +6 -4
  23. cognite/neat/_utils/reader/__init__.py +2 -2
  24. cognite/neat/_utils/reader/_base.py +40 -35
  25. cognite/neat/_utils/text.py +12 -0
  26. cognite/neat/_version.py +2 -2
  27. cognite_neat-0.102.0.dist-info/METADATA +113 -0
  28. {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/RECORD +31 -31
  29. cognite_neat-0.100.1.dist-info/METADATA +0 -215
  30. {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/LICENSE +0 -0
  31. {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/WHEEL +0 -0
  32. {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/entry_points.txt +0 -0
@@ -71,7 +71,7 @@ UNKNOWN_TYPE = DEFAULT_NAMESPACE.UnknownType
71
71
  XML_SCHEMA_NAMESPACE = Namespace("http://www.w3.org/2001/XMLSchema#")
72
72
 
73
73
 
74
- def get_default_prefixes() -> dict[str, Namespace]:
74
+ def get_default_prefixes_and_namespaces() -> dict[str, Namespace]:
75
75
  return {
76
76
  "owl": OWL._NS,
77
77
  "rdf": RDF._NS,
@@ -84,6 +84,10 @@ def get_default_prefixes() -> dict[str, Namespace]:
84
84
  "imf": Namespace("http://ns.imfid.org/imf#"),
85
85
  "pav": Namespace("http://purl.org/pav/"),
86
86
  "foaf": FOAF._NS,
87
+ "dexpi": Namespace("http://sandbox.dexpi.org/rdl/"),
88
+ "qudt": Namespace("https://qudt.org/vocab/unit/"),
89
+ "iodd": Namespace("http://www.io-link.com/IODD/2010/10/"),
90
+ "aml": Namespace("https://www.automationml.org/"),
87
91
  }
88
92
 
89
93
 
@@ -156,7 +156,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
156
156
 
157
157
  for identifier, properties in reader:
158
158
  try:
159
- print(view_id)
160
159
  yield self._create_node(identifier, properties, pydantic_cls, view_id)
161
160
  except ValueError as e:
162
161
  error_node = ResourceCreationError(identifier, "node", error=str(e))
@@ -198,7 +197,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
198
197
  json_fields: list[str] = []
199
198
  for prop_name, prop in view.properties.items():
200
199
  if isinstance(prop, dm.EdgeConnection):
201
- edge_by_property[prop.type.external_id] = prop_name, prop
200
+ edge_by_property[prop_name] = prop_name, prop
202
201
  if isinstance(prop, dm.MappedProperty):
203
202
  if isinstance(prop.type, dm.DirectRelation):
204
203
  direct_relation_by_property[prop_name] = prop.type
@@ -209,7 +209,10 @@ class Queries:
209
209
  elif (
210
210
  isinstance(object_, URIRef)
211
211
  and property_types
212
- and property_types.get(property_, None) == EntityTypes.object_property
212
+ and (
213
+ property_types.get(property_, None) == EntityTypes.object_property
214
+ or property_types.get(property_, None) == EntityTypes.undefined
215
+ )
213
216
  ):
214
217
  value = remove_namespace_from_uri(object_, validation="prefix")
215
218
 
@@ -233,7 +236,6 @@ class Queries:
233
236
  else:
234
237
  # we should not have multiple rdf:type values
235
238
  continue
236
-
237
239
  if property_values:
238
240
  return (
239
241
  identifier,
@@ -358,3 +360,21 @@ class Queries:
358
360
  dropped_types[t] = len(instance_ids)
359
361
  remove_instance_ids_in_batch(self.graph, instance_ids)
360
362
  return dropped_types
363
+
364
+ def multi_type_instances(self) -> dict[str, list[str]]:
365
+ """Find instances with multiple types"""
366
+
367
+ query = """
368
+ SELECT ?instance (GROUP_CONCAT(str(?type); SEPARATOR=",") AS ?types)
369
+ WHERE {
370
+ ?instance a ?type .
371
+ }
372
+ GROUP BY ?instance
373
+ HAVING (COUNT(?type) > 1)
374
+ """
375
+
376
+ result = {}
377
+ for instance, types in self.graph.query(query): # type: ignore
378
+ result[remove_namespace_from_uri(instance)] = remove_namespace_from_uri(types.split(","))
379
+
380
+ return result
@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
6
6
  from rdflib import Graph, Literal, Namespace
7
7
  from rdflib.term import URIRef
8
8
 
9
- from cognite.neat._constants import get_default_prefixes
9
+ from cognite.neat._constants import get_default_prefixes_and_namespaces
10
10
  from cognite.neat._rules.models._rdfpath import (
11
11
  Hop,
12
12
  Step,
@@ -51,7 +51,7 @@ def generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
51
51
  Prefix header
52
52
  """
53
53
 
54
- prefixes = prefixes or get_default_prefixes()
54
+ prefixes = prefixes or get_default_prefixes_and_namespaces()
55
55
 
56
56
  return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
57
57
 
@@ -81,7 +81,7 @@ def get_predicate_id(
81
81
  ID of predicate (aka property) connecting subject and object
82
82
  """
83
83
 
84
- prefixes = prefixes or get_default_prefixes()
84
+ prefixes = prefixes or get_default_prefixes_and_namespaces()
85
85
 
86
86
  query = """
87
87
 
@@ -123,7 +123,7 @@ def hop2property_path(
123
123
  str
124
124
  Property path string for hop traversal (e.g. ^rdf:type/rdfs:subClassOf)
125
125
  """
126
- prefixes = prefixes if prefixes else get_default_prefixes()
126
+ prefixes = prefixes if prefixes else get_default_prefixes_and_namespaces()
127
127
 
128
128
  # setting previous step to origin, as we are starting from there
129
129
  previous_step = Step(class_=hop.class_, direction="origin")
@@ -7,6 +7,13 @@ from ._classic_cdf import (
7
7
  AssetTimeSeriesConnector,
8
8
  RelationshipAsEdgeTransformer,
9
9
  )
10
+ from ._prune_graph import (
11
+ AttachPropertyFromTargetToSource,
12
+ PruneDanglingNodes,
13
+ PruneDeadEndEdges,
14
+ PruneInstancesOfUnknownType,
15
+ PruneTypes,
16
+ )
10
17
  from ._rdfpath import AddSelfReferenceProperty, MakeConnectionOnExactMatch
11
18
  from ._value_type import SplitMultiValueProperty
12
19
 
@@ -21,6 +28,11 @@ __all__ = [
21
28
  "SplitMultiValueProperty",
22
29
  "RelationshipAsEdgeTransformer",
23
30
  "MakeConnectionOnExactMatch",
31
+ "AttachPropertyFromTargetToSource",
32
+ "PruneDanglingNodes",
33
+ "PruneTypes",
34
+ "PruneDeadEndEdges",
35
+ "PruneInstancesOfUnknownType",
24
36
  ]
25
37
 
26
38
  Transformers = (
@@ -34,4 +46,9 @@ Transformers = (
34
46
  | SplitMultiValueProperty
35
47
  | RelationshipAsEdgeTransformer
36
48
  | MakeConnectionOnExactMatch
49
+ | AttachPropertyFromTargetToSource
50
+ | PruneDanglingNodes
51
+ | PruneTypes
52
+ | PruneDeadEndEdges
53
+ | PruneInstancesOfUnknownType
37
54
  )
@@ -6,7 +6,7 @@ from rdflib import Graph
6
6
 
7
7
  class BaseTransformer(ABC):
8
8
  description: str
9
- _use_only_once: bool
9
+ _use_only_once: bool = False
10
10
  _need_changes: ClassVar[frozenset[str]] = frozenset()
11
11
 
12
12
  @abstractmethod
@@ -2,12 +2,12 @@ from rdflib import Namespace
2
2
 
3
3
  from cognite.neat._graph.extractors import IODDExtractor
4
4
 
5
- from ._prune_graph import PruneDanglingNodes, TwoHopFlattener
5
+ from ._prune_graph import AttachPropertyFromTargetToSource, PruneDanglingNodes
6
6
 
7
7
  IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
8
8
 
9
9
 
10
- class IODDTwoHopFlattener(TwoHopFlattener):
10
+ class IODDAttachPropertyFromTargetToSource(AttachPropertyFromTargetToSource):
11
11
  _need_changes = frozenset(
12
12
  {
13
13
  str(IODDExtractor.__name__),
@@ -15,11 +15,16 @@ class IODDTwoHopFlattener(TwoHopFlattener):
15
15
  )
16
16
 
17
17
  def __init__(self):
18
- super().__init__(destination_node_type=IODD.TextObject, property_predicate=IODD.value, property_name="value")
18
+ super().__init__(
19
+ target_node_type=IODD.TextObject,
20
+ target_property=IODD.value,
21
+ delete_target_node=True,
22
+ namespace=IODD,
23
+ )
19
24
 
20
25
 
21
26
  class IODDPruneDanglingNodes(PruneDanglingNodes):
22
- _need_changes = frozenset({str(IODDExtractor.__name__), str(IODDTwoHopFlattener.__name__)})
27
+ _need_changes = frozenset({str(IODDExtractor.__name__), str(IODDAttachPropertyFromTargetToSource.__name__)})
23
28
 
24
29
  def __init__(self):
25
30
  super().__init__(node_prune_types=[IODD.TextObject])
@@ -1,84 +1,151 @@
1
+ from typing import cast
2
+
1
3
  from rdflib import Graph, Namespace, URIRef
2
- from rdflib.query import ResultRow
3
- from rdflib.term import Identifier
4
+
5
+ from cognite.neat._constants import DEFAULT_NAMESPACE
6
+ from cognite.neat._shared import Triple
7
+ from cognite.neat._utils.rdf_ import as_neat_compliant_uri
8
+ from cognite.neat._utils.text import sentence_or_string_to_camel
4
9
 
5
10
  from ._base import BaseTransformer
6
11
 
7
12
 
8
- # TODO: Handle the cse when value is None, which will not make the TextObject resolve
9
- class TwoHopFlattener(BaseTransformer):
13
+ class AttachPropertyFromTargetToSource(BaseTransformer):
10
14
  """
11
- Transformer that will flatten the distance between a source node, an intermediate connecting node, and a
12
- target property that is connected to the intermediate node.
13
- The transformation result is that the target property is attached directly to the source node, instead of having
14
- to go via the intermediate node.
15
- The user can also provide a flag to decide if the intermediate node should be removed from the graph or not
16
- after connecting the target property to the source node.
15
+ Transformer that considers a TargetNode and SourceNode relationship, to extract a property that is attached to
16
+ the TargetNode, and attaches it to the SourceNode instead, while also deleting the edge between
17
+ the SourceNode and TargetNode.
18
+ This means that you no longer have to go via the SourceNode to TargetNode to extract
19
+ the desired property from TargetNode, you can get it directly from the SourceNode instead.
20
+ Further, there are two ways of defining the predicate for the new property to attach to
21
+ the SourceNode. The predicate that is used will either be the old predicate between the SourceNode and TargetNode,
22
+ or, the TargetNode may hold a property with a value for the new predicate to use.
23
+ In this case, the user must specify the name of this predicate property connected to the TargetNode.
24
+ Consider the following example for illustration:
25
+
26
+ Ex. AttachPropertyFromTargetToSource
27
+ Graph before transformation:
28
+
29
+ :SourceNode a :SourceType .
30
+ :SourceNode :sourceProperty :TargetNode .
31
+
32
+ :TargetNode a :TargetType .
33
+ :TargetNode :propertyWhichValueWeWant 'Target Value' .
34
+ :TargetNode :propertyWhichValueWeMightWantAsNameForNewProperty 'PropertyName'
35
+
36
+ Use case A after transformation - attach new property to SourceNode using old predicate:
37
+
38
+ :SourceNode a :SourceType .
39
+ :SourceNode :sourceProperty 'Target Value' .
17
40
 
18
- Ex. TwoHopFlattener:
41
+ Use case B after transformation - extract new predicate from one of the properties of the TargetNode:
19
42
 
20
- Graph before flattening (with deletion of intermediate node):
21
- node(A, rdf:type(Pump)) -(predicate("vendor"))>
22
- node(B, rdf:type(TextObject)) -(predicate("value"))> Literal("CompanyX")
43
+ :SourceNode a :SourceType .
44
+ :SourceNode :PropertyName 'Target Value' .
23
45
 
24
- Graph after flattening nodes with destination_node_type = rdf:type(TextObject), property_predicate = :value,
25
- and property_name = "value":
26
46
 
27
- node(A, rdf:type(Pump)) -(predicate("vendor"))> Literal("CompanyX")
47
+ The user can provide a flag to decide if the intermediate target node should be removed from the graph or not
48
+ after connecting the target property to the source node. The example illustrates this.
49
+ The default however is False.
50
+
51
+ If delete_target_node is not set, the expected number of triples after this transformation should be the same as
52
+ before the transformation.
53
+
54
+ If delete_target_node is set, the expected number of triples should be:
55
+ #triples_before - #target_nodes * #target_nodes_properties
56
+
57
+ Number of triples after operation from above example: 5 - 1*3 = 2
28
58
 
29
59
  Args:
30
- destination_node_type: RDF.type of edge Node
31
- property_predicate: Predicate to use when resolving the value from the edge node
32
- property_name: name of the property that the intermediate node is pointing to
33
- delete_connecting_node: bool if the intermediate Node and Edge between source Node
34
- and target property should be deleted. Defaults to True.
60
+ target_node_type: RDF.type of edge Node
61
+ target_property: URIRef of the property that holds the value attached to the intermediate node
62
+ target_property_holding_new_property: URIRef of the property which value will be new
63
+ property that will be added to the source node
64
+ delete_target_node: bool if the intermediate Node and Edge between source Node
65
+ and target property should be deleted. Defaults to False.
66
+ convert_literal_to_uri: bool if the value of the new property should be converted to URIRef. Defaults to False.
67
+ namespace: Namespace to use when converting value to URIRef. Defaults to DEFAULT_NAMESPACE.
35
68
  """
36
69
 
37
- description: str = "Prunes the graph of specified node types that do not have connections to other nodes."
38
- _query_template: str = """SELECT ?sourceNode ?property ?destinationNode ?value WHERE {{
39
- ?sourceNode ?property ?destinationNode .
40
- ?destinationNode a <{destination_node_type}> .
41
- ?destinationNode <{property_predicate}> ?{property_name} . }}"""
70
+ description: str = "Attaches a target property from a target node that is connected to a source node."
71
+
72
+ _query_template_use_case_a: str = """
73
+ SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
74
+ ?sourceNode ?sourceProperty ?targetNode .
75
+ BIND( <{target_property}> as ?newSourceProperty ) .
76
+ ?targetNode a <{target_node_type}> .
77
+ ?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
78
+
79
+ _query_template_use_case_b: str = """
80
+ SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
81
+ ?sourceNode ?sourceProperty ?targetNode .
82
+ ?targetNode a <{target_node_type}> .
83
+ ?targetNode <{target_property_holding_new_property_name}> ?newSourceProperty .
84
+ ?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
42
85
 
43
86
  def __init__(
44
87
  self,
45
- destination_node_type: URIRef,
46
- property_predicate: Namespace,
47
- property_name: str,
48
- delete_connecting_node: bool = True,
88
+ target_node_type: URIRef,
89
+ target_property: URIRef,
90
+ target_property_holding_new_property: URIRef | None = None,
91
+ delete_target_node: bool = False,
92
+ convert_literal_to_uri: bool = False,
93
+ namespace: Namespace | None = None,
49
94
  ):
50
- self.destination_node_type = destination_node_type
51
- self.property_predicate = property_predicate
52
- self.property_name = property_name
53
- self.delete_connecting_node = delete_connecting_node
95
+ self.target_node_type = target_node_type
96
+ self.target_property = target_property
97
+ self.delete_target_node = delete_target_node
98
+ self.target_property_holding_new_property = target_property_holding_new_property
99
+ self.convert_literal_to_uri = convert_literal_to_uri
100
+ self.namespace = namespace or DEFAULT_NAMESPACE
54
101
 
55
- def transform(self, graph: Graph) -> None:
56
- nodes_to_delete: list[Identifier] = []
57
-
58
- graph_traversals = list(
59
- graph.query(
60
- self._query_template.format(
61
- destination_node_type=self.destination_node_type,
62
- property_predicate=self.property_predicate,
63
- property_name=self.property_name,
64
- )
102
+ def transform(self, graph) -> None:
103
+ nodes_to_delete: list[tuple] = []
104
+
105
+ if self.target_property_holding_new_property is not None:
106
+ query = self._query_template_use_case_b.format(
107
+ target_node_type=self.target_node_type,
108
+ target_property_holding_new_property_name=self.target_property_holding_new_property,
109
+ target_property=self.target_property,
110
+ )
111
+ else:
112
+ query = self._query_template_use_case_a.format(
113
+ target_node_type=self.target_node_type,
114
+ target_property=self.target_property,
65
115
  )
66
- )
67
116
 
68
- for path in graph_traversals:
69
- if isinstance(path, ResultRow):
70
- source_node, predicate, destination_node, property_value = path.asdict().values()
117
+ for (
118
+ source_node,
119
+ old_predicate,
120
+ target_node,
121
+ new_predicate_value,
122
+ new_property_value,
123
+ ) in graph.query(query):
124
+ if self.target_property_holding_new_property is not None:
125
+ # Ensure new predicate is URI compliant as we are creating a new predicate
126
+ new_predicate_value_string = sentence_or_string_to_camel(str(new_predicate_value))
127
+ predicate = as_neat_compliant_uri(self.namespace[new_predicate_value_string])
128
+ else:
129
+ predicate = old_predicate
130
+ # Create new connection from source node to value
131
+ graph.add(
132
+ (
133
+ source_node,
134
+ predicate,
135
+ (self.namespace[new_property_value] if self.convert_literal_to_uri else new_property_value),
136
+ )
137
+ )
138
+ # Remove old relationship between source node and destination node
139
+ graph.remove((source_node, old_predicate, target_node))
71
140
 
72
- # Create new connection from source node to value
73
- graph.add((source_node, predicate, property_value))
74
- nodes_to_delete.append(destination_node)
141
+ nodes_to_delete.append(target_node)
75
142
 
76
- if self.delete_connecting_node:
77
- for node in nodes_to_delete:
78
- # Remove edge triples to node
79
- graph.remove((None, None, node))
80
- # Remove node triple
81
- graph.remove((node, None, None))
143
+ if self.delete_target_node:
144
+ for target_node in nodes_to_delete:
145
+ # Remove triples with edges to target_node
146
+ graph.remove((None, None, target_node))
147
+ # Remove target node triple and its properties
148
+ graph.remove((target_node, None, None))
82
149
 
83
150
 
84
151
  class PruneDanglingNodes(BaseTransformer):
@@ -101,7 +168,7 @@ class PruneDanglingNodes(BaseTransformer):
101
168
  node_prune_types: list of RDF types to prune from the Graph if they are stand-alone Nodes
102
169
  """
103
170
 
104
- description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
171
+ description: str = "Prunes nodes of specific rdf types that do not have any connection to them."
105
172
  _query_template = """
106
173
  SELECT ?subject
107
174
  WHERE {{
@@ -117,10 +184,74 @@ class PruneDanglingNodes(BaseTransformer):
117
184
  self.node_prune_types = node_prune_types
118
185
 
119
186
  def transform(self, graph: Graph) -> None:
120
- for object_type in self.node_prune_types:
121
- nodes_without_neighbours = list(graph.query(self._query_template.format(rdf_type=object_type)))
187
+ for type_ in self.node_prune_types:
188
+ for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))): # type: ignore
189
+ graph.remove((subject, None, None))
190
+
191
+
192
+ class PruneTypes(BaseTransformer):
193
+ """
194
+ Removes all the instances of specific type
195
+ """
196
+
197
+ description: str = "Prunes nodes of specific rdf types"
198
+ _query_template = """
199
+ SELECT ?subject
200
+ WHERE {{
201
+ ?subject a <{rdf_type}> .
202
+ }}
203
+ """
204
+
205
+ def __init__(
206
+ self,
207
+ node_prune_types: list[URIRef],
208
+ ):
209
+ self.node_prune_types = node_prune_types
210
+
211
+ def transform(self, graph: Graph) -> None:
212
+ for type_ in self.node_prune_types:
213
+ for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))): # type: ignore
214
+ graph.remove((subject, None, None))
215
+
216
+
217
+ class PruneDeadEndEdges(BaseTransformer):
218
+ """
219
+ Removes all the triples where object is a node that is not found in graph
220
+ """
221
+
222
+ description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
223
+ _query_template = """
224
+ SELECT ?subject ?predicate ?object
225
+ WHERE {
226
+ ?subject ?predicate ?object .
227
+ FILTER (isIRI(?object) && ?predicate != rdf:type)
228
+ FILTER NOT EXISTS {?object ?p ?o .}
229
+
230
+ }
122
231
 
123
- for node in nodes_without_neighbours:
124
- # Remove node and its property triples in the graph
125
- if isinstance(node, ResultRow):
126
- graph.remove((node["subject"], None, None))
232
+ """
233
+
234
+ def transform(self, graph: Graph) -> None:
235
+ for triple in graph.query(self._query_template):
236
+ graph.remove(cast(Triple, triple))
237
+
238
+
239
+ class PruneInstancesOfUnknownType(BaseTransformer):
240
+ """
241
+ Removes all the triples where object is a node that is not found in graph
242
+ """
243
+
244
+ description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
245
+ _query_template = """
246
+ SELECT DISTINCT ?subject
247
+ WHERE {
248
+ ?subject ?p ?o .
249
+ FILTER NOT EXISTS {?subject a ?object .}
250
+
251
+ }
252
+
253
+ """
254
+
255
+ def transform(self, graph: Graph) -> None:
256
+ for (subject,) in graph.query(self._query_template): # type: ignore
257
+ graph.remove((subject, None, None))
@@ -12,6 +12,7 @@ from cognite.client.data_classes._base import (
12
12
  from cognite.client.data_classes.data_modeling import (
13
13
  DataModelApplyList,
14
14
  DataModelId,
15
+ SpaceApply,
15
16
  ViewApplyList,
16
17
  )
17
18
  from cognite.client.exceptions import CogniteAPIError
@@ -200,6 +201,32 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
200
201
  loader.resource_name for loader, categorized in categorized_items_by_loader.items() if categorized.to_update
201
202
  )
202
203
 
204
+ deleted_by_name: dict[str, UploadResult] = {}
205
+ if not is_failing:
206
+ # Deletion is done in reverse order to take care of dependencies
207
+ for loader, items in reversed(categorized_items_by_loader.items()):
208
+ issue_list = IssueList()
209
+
210
+ if items.resource_name == client.loaders.data_models.resource_name:
211
+ warning_list = self._validate(list(items.item_ids()), client)
212
+ issue_list.extend(warning_list)
213
+
214
+ results = UploadResult(loader.resource_name, issues=issue_list) # type: ignore[var-annotated]
215
+ if dry_run:
216
+ results.deleted.update(items.to_delete_ids)
217
+ else:
218
+ if items.to_delete_ids:
219
+ try:
220
+ deleted = loader.delete(items.to_delete_ids)
221
+ except MultiCogniteAPIError as e:
222
+ results.deleted.update([loader.get_id(item) for item in e.success])
223
+ results.failed_deleted.update([loader.get_id(item) for item in e.failed])
224
+ for error in e.errors:
225
+ results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
226
+ else:
227
+ results.deleted.update(deleted)
228
+ deleted_by_name[loader.resource_name] = results
229
+
203
230
  for loader, items in categorized_items_by_loader.items():
204
231
  issue_list = IssueList()
205
232
 
@@ -221,28 +248,21 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
221
248
 
222
249
  results.unchanged.update(items.unchanged_ids)
223
250
  results.skipped.update(items.to_skip_ids)
251
+ if delete_results := deleted_by_name.get(loader.resource_name):
252
+ results.deleted.update(delete_results.deleted)
253
+ results.failed_deleted.update(delete_results.failed_deleted)
254
+ results.error_messages.extend(delete_results.error_messages)
255
+
224
256
  if dry_run:
225
257
  if self.existing in ["update", "force"]:
226
258
  # Assume all changed are successful
227
259
  results.changed.update(items.to_update_ids)
228
260
  elif self.existing == "skip":
229
261
  results.skipped.update(items.to_update_ids)
230
- results.deleted.update(items.to_delete_ids)
231
262
  results.created.update(items.to_create_ids)
232
263
  yield results
233
264
  continue
234
265
 
235
- if items.to_delete_ids:
236
- try:
237
- deleted = loader.delete(items.to_delete_ids)
238
- except MultiCogniteAPIError as e:
239
- results.deleted.update([loader.get_id(item) for item in e.success])
240
- results.failed_deleted.update([loader.get_id(item) for item in e.failed])
241
- for error in e.errors:
242
- results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
243
- else:
244
- results.deleted.update(deleted)
245
-
246
266
  if items.to_create:
247
267
  try:
248
268
  created = loader.create(items.to_create)
@@ -308,7 +328,9 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
308
328
  cdf_item = cdf_item_by_id.get(item_id)
309
329
  if cdf_item is None:
310
330
  categorized.to_create.append(item)
311
- elif is_redeploying or self.existing == "recreate":
331
+ elif (is_redeploying or self.existing == "recreate") and not isinstance(item, SpaceApply):
332
+ # Spaces are not deleted, instead they are updated. Deleting a space is an expensive operation
333
+ # and are seldom needed. If you need to delete the space, it should be done in a different operation.
312
334
  if not self.drop_data and loader.has_data(item_id):
313
335
  categorized.to_skip.append(cdf_item)
314
336
  else:
@@ -34,8 +34,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
34
34
  Args:
35
35
  styling: The styling to use for the Excel file. Defaults to "default". See below for details
36
36
  on the different styles.
37
- output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
38
- this role formate before being written to excel. If not provided, the role from the rules will be used.
39
37
  new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
40
38
  rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
41
39
  new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
@@ -120,6 +118,10 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
120
118
  main_header = self._main_header_by_sheet_name[sheet_name]
121
119
  sheet.append([main_header] + [""] * (len(headers) - 1))
122
120
  sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
121
+ if headers[0] == "Neat ID":
122
+ # Move the Neat ID to the end of the columns
123
+ headers = headers[1:] + ["Neat ID"]
124
+
123
125
  sheet.append(headers)
124
126
 
125
127
  fill_colors = itertools.cycle(["CADCFC", "FFFFFF"])
@@ -127,6 +129,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
127
129
  last_class: str | None = None
128
130
  item: dict[str, Any]
129
131
  for item in dumped_rules.get(sheet_name) or []:
132
+ if "Neat ID" in item:
133
+ # Move the Neat ID to the end of the columns
134
+ item["Neat ID"] = item.pop("Neat ID")
130
135
  row = list(item.values())
131
136
  class_ = row[0]
132
137