cognite-neat 0.100.1__py3-none-any.whl → 0.102.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_constants.py +5 -1
- cognite/neat/_graph/loaders/_rdf2dms.py +1 -2
- cognite/neat/_graph/queries/_base.py +22 -2
- cognite/neat/_graph/queries/_shared.py +4 -4
- cognite/neat/_graph/transformers/__init__.py +17 -0
- cognite/neat/_graph/transformers/_base.py +1 -1
- cognite/neat/_graph/transformers/_iodd.py +9 -4
- cognite/neat/_graph/transformers/_prune_graph.py +196 -65
- cognite/neat/_rules/exporters/_rules2dms.py +35 -13
- cognite/neat/_rules/exporters/_rules2excel.py +7 -2
- cognite/neat/_rules/importers/_dms2rules.py +51 -19
- cognite/neat/_rules/importers/_rdf/_base.py +2 -2
- cognite/neat/_rules/models/_base_rules.py +13 -9
- cognite/neat/_rules/models/dms/_rules.py +111 -39
- cognite/neat/_rules/models/information/_rules.py +52 -19
- cognite/neat/_session/_base.py +18 -0
- cognite/neat/_session/_prepare.py +85 -2
- cognite/neat/_session/_read.py +3 -3
- cognite/neat/_session/_to.py +1 -1
- cognite/neat/_session/engine/_load.py +3 -1
- cognite/neat/_store/_base.py +23 -2
- cognite/neat/_utils/auth.py +6 -4
- cognite/neat/_utils/reader/__init__.py +2 -2
- cognite/neat/_utils/reader/_base.py +40 -35
- cognite/neat/_utils/text.py +12 -0
- cognite/neat/_version.py +2 -2
- cognite_neat-0.102.0.dist-info/METADATA +113 -0
- {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/RECORD +31 -31
- cognite_neat-0.100.1.dist-info/METADATA +0 -215
- {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.100.1.dist-info → cognite_neat-0.102.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_constants.py
CHANGED
|
@@ -71,7 +71,7 @@ UNKNOWN_TYPE = DEFAULT_NAMESPACE.UnknownType
|
|
|
71
71
|
XML_SCHEMA_NAMESPACE = Namespace("http://www.w3.org/2001/XMLSchema#")
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def
|
|
74
|
+
def get_default_prefixes_and_namespaces() -> dict[str, Namespace]:
|
|
75
75
|
return {
|
|
76
76
|
"owl": OWL._NS,
|
|
77
77
|
"rdf": RDF._NS,
|
|
@@ -84,6 +84,10 @@ def get_default_prefixes() -> dict[str, Namespace]:
|
|
|
84
84
|
"imf": Namespace("http://ns.imfid.org/imf#"),
|
|
85
85
|
"pav": Namespace("http://purl.org/pav/"),
|
|
86
86
|
"foaf": FOAF._NS,
|
|
87
|
+
"dexpi": Namespace("http://sandbox.dexpi.org/rdl/"),
|
|
88
|
+
"qudt": Namespace("https://qudt.org/vocab/unit/"),
|
|
89
|
+
"iodd": Namespace("http://www.io-link.com/IODD/2010/10/"),
|
|
90
|
+
"aml": Namespace("https://www.automationml.org/"),
|
|
87
91
|
}
|
|
88
92
|
|
|
89
93
|
|
|
@@ -156,7 +156,6 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
|
|
|
156
156
|
|
|
157
157
|
for identifier, properties in reader:
|
|
158
158
|
try:
|
|
159
|
-
print(view_id)
|
|
160
159
|
yield self._create_node(identifier, properties, pydantic_cls, view_id)
|
|
161
160
|
except ValueError as e:
|
|
162
161
|
error_node = ResourceCreationError(identifier, "node", error=str(e))
|
|
@@ -198,7 +197,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
|
|
|
198
197
|
json_fields: list[str] = []
|
|
199
198
|
for prop_name, prop in view.properties.items():
|
|
200
199
|
if isinstance(prop, dm.EdgeConnection):
|
|
201
|
-
edge_by_property[
|
|
200
|
+
edge_by_property[prop_name] = prop_name, prop
|
|
202
201
|
if isinstance(prop, dm.MappedProperty):
|
|
203
202
|
if isinstance(prop.type, dm.DirectRelation):
|
|
204
203
|
direct_relation_by_property[prop_name] = prop.type
|
|
@@ -209,7 +209,10 @@ class Queries:
|
|
|
209
209
|
elif (
|
|
210
210
|
isinstance(object_, URIRef)
|
|
211
211
|
and property_types
|
|
212
|
-
and
|
|
212
|
+
and (
|
|
213
|
+
property_types.get(property_, None) == EntityTypes.object_property
|
|
214
|
+
or property_types.get(property_, None) == EntityTypes.undefined
|
|
215
|
+
)
|
|
213
216
|
):
|
|
214
217
|
value = remove_namespace_from_uri(object_, validation="prefix")
|
|
215
218
|
|
|
@@ -233,7 +236,6 @@ class Queries:
|
|
|
233
236
|
else:
|
|
234
237
|
# we should not have multiple rdf:type values
|
|
235
238
|
continue
|
|
236
|
-
|
|
237
239
|
if property_values:
|
|
238
240
|
return (
|
|
239
241
|
identifier,
|
|
@@ -358,3 +360,21 @@ class Queries:
|
|
|
358
360
|
dropped_types[t] = len(instance_ids)
|
|
359
361
|
remove_instance_ids_in_batch(self.graph, instance_ids)
|
|
360
362
|
return dropped_types
|
|
363
|
+
|
|
364
|
+
def multi_type_instances(self) -> dict[str, list[str]]:
|
|
365
|
+
"""Find instances with multiple types"""
|
|
366
|
+
|
|
367
|
+
query = """
|
|
368
|
+
SELECT ?instance (GROUP_CONCAT(str(?type); SEPARATOR=",") AS ?types)
|
|
369
|
+
WHERE {
|
|
370
|
+
?instance a ?type .
|
|
371
|
+
}
|
|
372
|
+
GROUP BY ?instance
|
|
373
|
+
HAVING (COUNT(?type) > 1)
|
|
374
|
+
"""
|
|
375
|
+
|
|
376
|
+
result = {}
|
|
377
|
+
for instance, types in self.graph.query(query): # type: ignore
|
|
378
|
+
result[remove_namespace_from_uri(instance)] = remove_namespace_from_uri(types.split(","))
|
|
379
|
+
|
|
380
|
+
return result
|
|
@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
|
|
6
6
|
from rdflib import Graph, Literal, Namespace
|
|
7
7
|
from rdflib.term import URIRef
|
|
8
8
|
|
|
9
|
-
from cognite.neat._constants import
|
|
9
|
+
from cognite.neat._constants import get_default_prefixes_and_namespaces
|
|
10
10
|
from cognite.neat._rules.models._rdfpath import (
|
|
11
11
|
Hop,
|
|
12
12
|
Step,
|
|
@@ -51,7 +51,7 @@ def generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
|
|
|
51
51
|
Prefix header
|
|
52
52
|
"""
|
|
53
53
|
|
|
54
|
-
prefixes = prefixes or
|
|
54
|
+
prefixes = prefixes or get_default_prefixes_and_namespaces()
|
|
55
55
|
|
|
56
56
|
return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
|
|
57
57
|
|
|
@@ -81,7 +81,7 @@ def get_predicate_id(
|
|
|
81
81
|
ID of predicate (aka property) connecting subject and object
|
|
82
82
|
"""
|
|
83
83
|
|
|
84
|
-
prefixes = prefixes or
|
|
84
|
+
prefixes = prefixes or get_default_prefixes_and_namespaces()
|
|
85
85
|
|
|
86
86
|
query = """
|
|
87
87
|
|
|
@@ -123,7 +123,7 @@ def hop2property_path(
|
|
|
123
123
|
str
|
|
124
124
|
Property path string for hop traversal (e.g. ^rdf:type/rdfs:subClassOf)
|
|
125
125
|
"""
|
|
126
|
-
prefixes = prefixes if prefixes else
|
|
126
|
+
prefixes = prefixes if prefixes else get_default_prefixes_and_namespaces()
|
|
127
127
|
|
|
128
128
|
# setting previous step to origin, as we are starting from there
|
|
129
129
|
previous_step = Step(class_=hop.class_, direction="origin")
|
|
@@ -7,6 +7,13 @@ from ._classic_cdf import (
|
|
|
7
7
|
AssetTimeSeriesConnector,
|
|
8
8
|
RelationshipAsEdgeTransformer,
|
|
9
9
|
)
|
|
10
|
+
from ._prune_graph import (
|
|
11
|
+
AttachPropertyFromTargetToSource,
|
|
12
|
+
PruneDanglingNodes,
|
|
13
|
+
PruneDeadEndEdges,
|
|
14
|
+
PruneInstancesOfUnknownType,
|
|
15
|
+
PruneTypes,
|
|
16
|
+
)
|
|
10
17
|
from ._rdfpath import AddSelfReferenceProperty, MakeConnectionOnExactMatch
|
|
11
18
|
from ._value_type import SplitMultiValueProperty
|
|
12
19
|
|
|
@@ -21,6 +28,11 @@ __all__ = [
|
|
|
21
28
|
"SplitMultiValueProperty",
|
|
22
29
|
"RelationshipAsEdgeTransformer",
|
|
23
30
|
"MakeConnectionOnExactMatch",
|
|
31
|
+
"AttachPropertyFromTargetToSource",
|
|
32
|
+
"PruneDanglingNodes",
|
|
33
|
+
"PruneTypes",
|
|
34
|
+
"PruneDeadEndEdges",
|
|
35
|
+
"PruneInstancesOfUnknownType",
|
|
24
36
|
]
|
|
25
37
|
|
|
26
38
|
Transformers = (
|
|
@@ -34,4 +46,9 @@ Transformers = (
|
|
|
34
46
|
| SplitMultiValueProperty
|
|
35
47
|
| RelationshipAsEdgeTransformer
|
|
36
48
|
| MakeConnectionOnExactMatch
|
|
49
|
+
| AttachPropertyFromTargetToSource
|
|
50
|
+
| PruneDanglingNodes
|
|
51
|
+
| PruneTypes
|
|
52
|
+
| PruneDeadEndEdges
|
|
53
|
+
| PruneInstancesOfUnknownType
|
|
37
54
|
)
|
|
@@ -2,12 +2,12 @@ from rdflib import Namespace
|
|
|
2
2
|
|
|
3
3
|
from cognite.neat._graph.extractors import IODDExtractor
|
|
4
4
|
|
|
5
|
-
from ._prune_graph import
|
|
5
|
+
from ._prune_graph import AttachPropertyFromTargetToSource, PruneDanglingNodes
|
|
6
6
|
|
|
7
7
|
IODD = Namespace("http://www.io-link.com/IODD/2010/10/")
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
class
|
|
10
|
+
class IODDAttachPropertyFromTargetToSource(AttachPropertyFromTargetToSource):
|
|
11
11
|
_need_changes = frozenset(
|
|
12
12
|
{
|
|
13
13
|
str(IODDExtractor.__name__),
|
|
@@ -15,11 +15,16 @@ class IODDTwoHopFlattener(TwoHopFlattener):
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
def __init__(self):
|
|
18
|
-
super().__init__(
|
|
18
|
+
super().__init__(
|
|
19
|
+
target_node_type=IODD.TextObject,
|
|
20
|
+
target_property=IODD.value,
|
|
21
|
+
delete_target_node=True,
|
|
22
|
+
namespace=IODD,
|
|
23
|
+
)
|
|
19
24
|
|
|
20
25
|
|
|
21
26
|
class IODDPruneDanglingNodes(PruneDanglingNodes):
|
|
22
|
-
_need_changes = frozenset({str(IODDExtractor.__name__), str(
|
|
27
|
+
_need_changes = frozenset({str(IODDExtractor.__name__), str(IODDAttachPropertyFromTargetToSource.__name__)})
|
|
23
28
|
|
|
24
29
|
def __init__(self):
|
|
25
30
|
super().__init__(node_prune_types=[IODD.TextObject])
|
|
@@ -1,84 +1,151 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
1
3
|
from rdflib import Graph, Namespace, URIRef
|
|
2
|
-
|
|
3
|
-
from
|
|
4
|
+
|
|
5
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
6
|
+
from cognite.neat._shared import Triple
|
|
7
|
+
from cognite.neat._utils.rdf_ import as_neat_compliant_uri
|
|
8
|
+
from cognite.neat._utils.text import sentence_or_string_to_camel
|
|
4
9
|
|
|
5
10
|
from ._base import BaseTransformer
|
|
6
11
|
|
|
7
12
|
|
|
8
|
-
|
|
9
|
-
class TwoHopFlattener(BaseTransformer):
|
|
13
|
+
class AttachPropertyFromTargetToSource(BaseTransformer):
|
|
10
14
|
"""
|
|
11
|
-
Transformer that
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
to go via the
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
Transformer that considers a TargetNode and SourceNode relationship, to extract a property that is attached to
|
|
16
|
+
the TargetNode, and attaches it to the SourceNode instead, while also deleting the edge between
|
|
17
|
+
the SourceNode and TargetNode.
|
|
18
|
+
This means that you no longer have to go via the SourceNode to TargetNode to extract
|
|
19
|
+
the desired property from TargetNode, you can get it directly from the SourceNode instead.
|
|
20
|
+
Further, there are two ways of defining the predicate for the new property to attach to
|
|
21
|
+
the SourceNode. The predicate that is used will either be the old predicate between the SourceNode and TargetNode,
|
|
22
|
+
or, the TargetNode may hold a property with a value for the new predicate to use.
|
|
23
|
+
In this case, the user must specify the name of this predicate property connected to the TargetNode.
|
|
24
|
+
Consider the following example for illustration:
|
|
25
|
+
|
|
26
|
+
Ex. AttachPropertyFromTargetToSource
|
|
27
|
+
Graph before transformation:
|
|
28
|
+
|
|
29
|
+
:SourceNode a :SourceType .
|
|
30
|
+
:SourceNode :sourceProperty :TargetNode .
|
|
31
|
+
|
|
32
|
+
:TargetNode a :TargetType .
|
|
33
|
+
:TargetNode :propertyWhichValueWeWant 'Target Value' .
|
|
34
|
+
:TargetNode :propertyWhichValueWeMightWantAsNameForNewProperty 'PropertyName'
|
|
35
|
+
|
|
36
|
+
Use case A after transformation - attach new property to SourceNode using old predicate:
|
|
37
|
+
|
|
38
|
+
:SourceNode a :SourceType .
|
|
39
|
+
:SourceNode :sourceProperty 'Target Value' .
|
|
17
40
|
|
|
18
|
-
|
|
41
|
+
Use case B after transformation - extract new predicate from one of the properties of the TargetNode:
|
|
19
42
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
node(B, rdf:type(TextObject)) -(predicate("value"))> Literal("CompanyX")
|
|
43
|
+
:SourceNode a :SourceType .
|
|
44
|
+
:SourceNode :PropertyName 'Target Value' .
|
|
23
45
|
|
|
24
|
-
Graph after flattening nodes with destination_node_type = rdf:type(TextObject), property_predicate = :value,
|
|
25
|
-
and property_name = "value":
|
|
26
46
|
|
|
27
|
-
|
|
47
|
+
The user can provide a flag to decide if the intermediate target node should be removed from the graph or not
|
|
48
|
+
after connecting the target property to the source node. The example illustrates this.
|
|
49
|
+
The default however is False.
|
|
50
|
+
|
|
51
|
+
If delete_target_node is not set, the expected number of triples after this transformation should be the same as
|
|
52
|
+
before the transformation.
|
|
53
|
+
|
|
54
|
+
If delete_target_node is set, the expected number of triples should be:
|
|
55
|
+
#triples_before - #target_nodes * #target_nodes_properties
|
|
56
|
+
|
|
57
|
+
Number of triples after operation from above example: 5 - 1*3 = 2
|
|
28
58
|
|
|
29
59
|
Args:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
60
|
+
target_node_type: RDF.type of edge Node
|
|
61
|
+
target_property: URIRef of the property that holds the value attached to the intermediate node
|
|
62
|
+
target_property_holding_new_property: URIRef of the property which value will be new
|
|
63
|
+
property that will be added to the source node
|
|
64
|
+
delete_target_node: bool if the intermediate Node and Edge between source Node
|
|
65
|
+
and target property should be deleted. Defaults to False.
|
|
66
|
+
convert_literal_to_uri: bool if the value of the new property should be converted to URIRef. Defaults to False.
|
|
67
|
+
namespace: Namespace to use when converting value to URIRef. Defaults to DEFAULT_NAMESPACE.
|
|
35
68
|
"""
|
|
36
69
|
|
|
37
|
-
description: str = "
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
70
|
+
description: str = "Attaches a target property from a target node that is connected to a source node."
|
|
71
|
+
|
|
72
|
+
_query_template_use_case_a: str = """
|
|
73
|
+
SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
|
|
74
|
+
?sourceNode ?sourceProperty ?targetNode .
|
|
75
|
+
BIND( <{target_property}> as ?newSourceProperty ) .
|
|
76
|
+
?targetNode a <{target_node_type}> .
|
|
77
|
+
?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
|
|
78
|
+
|
|
79
|
+
_query_template_use_case_b: str = """
|
|
80
|
+
SELECT ?sourceNode ?sourceProperty ?targetNode ?newSourceProperty ?newSourcePropertyValue WHERE {{
|
|
81
|
+
?sourceNode ?sourceProperty ?targetNode .
|
|
82
|
+
?targetNode a <{target_node_type}> .
|
|
83
|
+
?targetNode <{target_property_holding_new_property_name}> ?newSourceProperty .
|
|
84
|
+
?targetNode <{target_property}> ?newSourcePropertyValue . }}"""
|
|
42
85
|
|
|
43
86
|
def __init__(
|
|
44
87
|
self,
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
88
|
+
target_node_type: URIRef,
|
|
89
|
+
target_property: URIRef,
|
|
90
|
+
target_property_holding_new_property: URIRef | None = None,
|
|
91
|
+
delete_target_node: bool = False,
|
|
92
|
+
convert_literal_to_uri: bool = False,
|
|
93
|
+
namespace: Namespace | None = None,
|
|
49
94
|
):
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
52
|
-
self.
|
|
53
|
-
self.
|
|
95
|
+
self.target_node_type = target_node_type
|
|
96
|
+
self.target_property = target_property
|
|
97
|
+
self.delete_target_node = delete_target_node
|
|
98
|
+
self.target_property_holding_new_property = target_property_holding_new_property
|
|
99
|
+
self.convert_literal_to_uri = convert_literal_to_uri
|
|
100
|
+
self.namespace = namespace or DEFAULT_NAMESPACE
|
|
54
101
|
|
|
55
|
-
def transform(self, graph
|
|
56
|
-
nodes_to_delete: list[
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
self.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
102
|
+
def transform(self, graph) -> None:
|
|
103
|
+
nodes_to_delete: list[tuple] = []
|
|
104
|
+
|
|
105
|
+
if self.target_property_holding_new_property is not None:
|
|
106
|
+
query = self._query_template_use_case_b.format(
|
|
107
|
+
target_node_type=self.target_node_type,
|
|
108
|
+
target_property_holding_new_property_name=self.target_property_holding_new_property,
|
|
109
|
+
target_property=self.target_property,
|
|
110
|
+
)
|
|
111
|
+
else:
|
|
112
|
+
query = self._query_template_use_case_a.format(
|
|
113
|
+
target_node_type=self.target_node_type,
|
|
114
|
+
target_property=self.target_property,
|
|
65
115
|
)
|
|
66
|
-
)
|
|
67
116
|
|
|
68
|
-
for
|
|
69
|
-
|
|
70
|
-
|
|
117
|
+
for (
|
|
118
|
+
source_node,
|
|
119
|
+
old_predicate,
|
|
120
|
+
target_node,
|
|
121
|
+
new_predicate_value,
|
|
122
|
+
new_property_value,
|
|
123
|
+
) in graph.query(query):
|
|
124
|
+
if self.target_property_holding_new_property is not None:
|
|
125
|
+
# Ensure new predicate is URI compliant as we are creating a new predicate
|
|
126
|
+
new_predicate_value_string = sentence_or_string_to_camel(str(new_predicate_value))
|
|
127
|
+
predicate = as_neat_compliant_uri(self.namespace[new_predicate_value_string])
|
|
128
|
+
else:
|
|
129
|
+
predicate = old_predicate
|
|
130
|
+
# Create new connection from source node to value
|
|
131
|
+
graph.add(
|
|
132
|
+
(
|
|
133
|
+
source_node,
|
|
134
|
+
predicate,
|
|
135
|
+
(self.namespace[new_property_value] if self.convert_literal_to_uri else new_property_value),
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
# Remove old relationship between source node and destination node
|
|
139
|
+
graph.remove((source_node, old_predicate, target_node))
|
|
71
140
|
|
|
72
|
-
|
|
73
|
-
graph.add((source_node, predicate, property_value))
|
|
74
|
-
nodes_to_delete.append(destination_node)
|
|
141
|
+
nodes_to_delete.append(target_node)
|
|
75
142
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
143
|
+
if self.delete_target_node:
|
|
144
|
+
for target_node in nodes_to_delete:
|
|
145
|
+
# Remove triples with edges to target_node
|
|
146
|
+
graph.remove((None, None, target_node))
|
|
147
|
+
# Remove target node triple and its properties
|
|
148
|
+
graph.remove((target_node, None, None))
|
|
82
149
|
|
|
83
150
|
|
|
84
151
|
class PruneDanglingNodes(BaseTransformer):
|
|
@@ -101,7 +168,7 @@ class PruneDanglingNodes(BaseTransformer):
|
|
|
101
168
|
node_prune_types: list of RDF types to prune from the Graph if they are stand-alone Nodes
|
|
102
169
|
"""
|
|
103
170
|
|
|
104
|
-
description: str = "Prunes
|
|
171
|
+
description: str = "Prunes nodes of specific rdf types that do not have any connection to them."
|
|
105
172
|
_query_template = """
|
|
106
173
|
SELECT ?subject
|
|
107
174
|
WHERE {{
|
|
@@ -117,10 +184,74 @@ class PruneDanglingNodes(BaseTransformer):
|
|
|
117
184
|
self.node_prune_types = node_prune_types
|
|
118
185
|
|
|
119
186
|
def transform(self, graph: Graph) -> None:
|
|
120
|
-
for
|
|
121
|
-
|
|
187
|
+
for type_ in self.node_prune_types:
|
|
188
|
+
for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))): # type: ignore
|
|
189
|
+
graph.remove((subject, None, None))
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class PruneTypes(BaseTransformer):
|
|
193
|
+
"""
|
|
194
|
+
Removes all the instances of specific type
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
description: str = "Prunes nodes of specific rdf types"
|
|
198
|
+
_query_template = """
|
|
199
|
+
SELECT ?subject
|
|
200
|
+
WHERE {{
|
|
201
|
+
?subject a <{rdf_type}> .
|
|
202
|
+
}}
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
def __init__(
|
|
206
|
+
self,
|
|
207
|
+
node_prune_types: list[URIRef],
|
|
208
|
+
):
|
|
209
|
+
self.node_prune_types = node_prune_types
|
|
210
|
+
|
|
211
|
+
def transform(self, graph: Graph) -> None:
|
|
212
|
+
for type_ in self.node_prune_types:
|
|
213
|
+
for (subject,) in list(graph.query(self._query_template.format(rdf_type=type_))): # type: ignore
|
|
214
|
+
graph.remove((subject, None, None))
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class PruneDeadEndEdges(BaseTransformer):
|
|
218
|
+
"""
|
|
219
|
+
Removes all the triples where object is a node that is not found in graph
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
|
|
223
|
+
_query_template = """
|
|
224
|
+
SELECT ?subject ?predicate ?object
|
|
225
|
+
WHERE {
|
|
226
|
+
?subject ?predicate ?object .
|
|
227
|
+
FILTER (isIRI(?object) && ?predicate != rdf:type)
|
|
228
|
+
FILTER NOT EXISTS {?object ?p ?o .}
|
|
229
|
+
|
|
230
|
+
}
|
|
122
231
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
232
|
+
"""
|
|
233
|
+
|
|
234
|
+
def transform(self, graph: Graph) -> None:
|
|
235
|
+
for triple in graph.query(self._query_template):
|
|
236
|
+
graph.remove(cast(Triple, triple))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class PruneInstancesOfUnknownType(BaseTransformer):
|
|
240
|
+
"""
|
|
241
|
+
Removes all the triples where object is a node that is not found in graph
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
description: str = "Prunes the graph of specified rdf types that do not have connections to other nodes."
|
|
245
|
+
_query_template = """
|
|
246
|
+
SELECT DISTINCT ?subject
|
|
247
|
+
WHERE {
|
|
248
|
+
?subject ?p ?o .
|
|
249
|
+
FILTER NOT EXISTS {?subject a ?object .}
|
|
250
|
+
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
def transform(self, graph: Graph) -> None:
|
|
256
|
+
for (subject,) in graph.query(self._query_template): # type: ignore
|
|
257
|
+
graph.remove((subject, None, None))
|
|
@@ -12,6 +12,7 @@ from cognite.client.data_classes._base import (
|
|
|
12
12
|
from cognite.client.data_classes.data_modeling import (
|
|
13
13
|
DataModelApplyList,
|
|
14
14
|
DataModelId,
|
|
15
|
+
SpaceApply,
|
|
15
16
|
ViewApplyList,
|
|
16
17
|
)
|
|
17
18
|
from cognite.client.exceptions import CogniteAPIError
|
|
@@ -200,6 +201,32 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
200
201
|
loader.resource_name for loader, categorized in categorized_items_by_loader.items() if categorized.to_update
|
|
201
202
|
)
|
|
202
203
|
|
|
204
|
+
deleted_by_name: dict[str, UploadResult] = {}
|
|
205
|
+
if not is_failing:
|
|
206
|
+
# Deletion is done in reverse order to take care of dependencies
|
|
207
|
+
for loader, items in reversed(categorized_items_by_loader.items()):
|
|
208
|
+
issue_list = IssueList()
|
|
209
|
+
|
|
210
|
+
if items.resource_name == client.loaders.data_models.resource_name:
|
|
211
|
+
warning_list = self._validate(list(items.item_ids()), client)
|
|
212
|
+
issue_list.extend(warning_list)
|
|
213
|
+
|
|
214
|
+
results = UploadResult(loader.resource_name, issues=issue_list) # type: ignore[var-annotated]
|
|
215
|
+
if dry_run:
|
|
216
|
+
results.deleted.update(items.to_delete_ids)
|
|
217
|
+
else:
|
|
218
|
+
if items.to_delete_ids:
|
|
219
|
+
try:
|
|
220
|
+
deleted = loader.delete(items.to_delete_ids)
|
|
221
|
+
except MultiCogniteAPIError as e:
|
|
222
|
+
results.deleted.update([loader.get_id(item) for item in e.success])
|
|
223
|
+
results.failed_deleted.update([loader.get_id(item) for item in e.failed])
|
|
224
|
+
for error in e.errors:
|
|
225
|
+
results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
|
|
226
|
+
else:
|
|
227
|
+
results.deleted.update(deleted)
|
|
228
|
+
deleted_by_name[loader.resource_name] = results
|
|
229
|
+
|
|
203
230
|
for loader, items in categorized_items_by_loader.items():
|
|
204
231
|
issue_list = IssueList()
|
|
205
232
|
|
|
@@ -221,28 +248,21 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
221
248
|
|
|
222
249
|
results.unchanged.update(items.unchanged_ids)
|
|
223
250
|
results.skipped.update(items.to_skip_ids)
|
|
251
|
+
if delete_results := deleted_by_name.get(loader.resource_name):
|
|
252
|
+
results.deleted.update(delete_results.deleted)
|
|
253
|
+
results.failed_deleted.update(delete_results.failed_deleted)
|
|
254
|
+
results.error_messages.extend(delete_results.error_messages)
|
|
255
|
+
|
|
224
256
|
if dry_run:
|
|
225
257
|
if self.existing in ["update", "force"]:
|
|
226
258
|
# Assume all changed are successful
|
|
227
259
|
results.changed.update(items.to_update_ids)
|
|
228
260
|
elif self.existing == "skip":
|
|
229
261
|
results.skipped.update(items.to_update_ids)
|
|
230
|
-
results.deleted.update(items.to_delete_ids)
|
|
231
262
|
results.created.update(items.to_create_ids)
|
|
232
263
|
yield results
|
|
233
264
|
continue
|
|
234
265
|
|
|
235
|
-
if items.to_delete_ids:
|
|
236
|
-
try:
|
|
237
|
-
deleted = loader.delete(items.to_delete_ids)
|
|
238
|
-
except MultiCogniteAPIError as e:
|
|
239
|
-
results.deleted.update([loader.get_id(item) for item in e.success])
|
|
240
|
-
results.failed_deleted.update([loader.get_id(item) for item in e.failed])
|
|
241
|
-
for error in e.errors:
|
|
242
|
-
results.error_messages.append(f"Failed to delete {loader.resource_name}: {error!s}")
|
|
243
|
-
else:
|
|
244
|
-
results.deleted.update(deleted)
|
|
245
|
-
|
|
246
266
|
if items.to_create:
|
|
247
267
|
try:
|
|
248
268
|
created = loader.create(items.to_create)
|
|
@@ -308,7 +328,9 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
308
328
|
cdf_item = cdf_item_by_id.get(item_id)
|
|
309
329
|
if cdf_item is None:
|
|
310
330
|
categorized.to_create.append(item)
|
|
311
|
-
elif is_redeploying or self.existing == "recreate":
|
|
331
|
+
elif (is_redeploying or self.existing == "recreate") and not isinstance(item, SpaceApply):
|
|
332
|
+
# Spaces are not deleted, instead they are updated. Deleting a space is an expensive operation
|
|
333
|
+
# and are seldom needed. If you need to delete the space, it should be done in a different operation.
|
|
312
334
|
if not self.drop_data and loader.has_data(item_id):
|
|
313
335
|
categorized.to_skip.append(cdf_item)
|
|
314
336
|
else:
|
|
@@ -34,8 +34,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
34
34
|
Args:
|
|
35
35
|
styling: The styling to use for the Excel file. Defaults to "default". See below for details
|
|
36
36
|
on the different styles.
|
|
37
|
-
output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
|
|
38
|
-
this role formate before being written to excel. If not provided, the role from the rules will be used.
|
|
39
37
|
new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
|
|
40
38
|
rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
|
|
41
39
|
new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
|
|
@@ -120,6 +118,10 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
120
118
|
main_header = self._main_header_by_sheet_name[sheet_name]
|
|
121
119
|
sheet.append([main_header] + [""] * (len(headers) - 1))
|
|
122
120
|
sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
|
|
121
|
+
if headers[0] == "Neat ID":
|
|
122
|
+
# Move the Neat ID to the end of the columns
|
|
123
|
+
headers = headers[1:] + ["Neat ID"]
|
|
124
|
+
|
|
123
125
|
sheet.append(headers)
|
|
124
126
|
|
|
125
127
|
fill_colors = itertools.cycle(["CADCFC", "FFFFFF"])
|
|
@@ -127,6 +129,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
127
129
|
last_class: str | None = None
|
|
128
130
|
item: dict[str, Any]
|
|
129
131
|
for item in dumped_rules.get(sheet_name) or []:
|
|
132
|
+
if "Neat ID" in item:
|
|
133
|
+
# Move the Neat ID to the end of the columns
|
|
134
|
+
item["Neat ID"] = item.pop("Neat ID")
|
|
130
135
|
row = list(item.values())
|
|
131
136
|
class_ = row[0]
|
|
132
137
|
|