cognite-neat 0.86.0__py3-none-any.whl → 0.87.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_mock_graph_generator.py +7 -8
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +94 -20
- cognite/neat/graph/loaders/_rdf2dms.py +1 -1
- cognite/neat/graph/queries/_base.py +1 -1
- cognite/neat/graph/queries/_construct.py +2 -2
- cognite/neat/graph/queries/_shared.py +20 -6
- cognite/neat/graph/stores/_base.py +4 -3
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +48 -15
- cognite/neat/legacy/rules/importers/_graph2rules.py +34 -7
- cognite/neat/legacy/rules/models/raw_rules.py +18 -6
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +128 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +155 -0
- cognite/neat/rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +2 -2
- cognite/neat/rules/models/_base.py +7 -7
- cognite/neat/rules/models/asset/_rules.py +4 -5
- cognite/neat/rules/models/dms/_converter.py +1 -2
- cognite/neat/rules/models/dms/_rules.py +3 -0
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +2 -9
- cognite/neat/rules/models/information/_rules.py +10 -8
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +129 -27
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/RECORD +46 -45
- cognite/neat/rules/analysis/_information_rules.py +0 -476
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,13 +1,392 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import warnings
|
|
1
3
|
from abc import ABC, abstractmethod
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from collections.abc import Set
|
|
6
|
+
from dataclasses import dataclass
|
|
2
7
|
from typing import Generic, TypeVar
|
|
3
8
|
|
|
4
|
-
|
|
5
|
-
from
|
|
9
|
+
import pandas as pd
|
|
10
|
+
from pydantic import BaseModel
|
|
6
11
|
|
|
7
|
-
|
|
12
|
+
from cognite.neat.rules.models._base import BaseRules
|
|
13
|
+
from cognite.neat.rules.models._rdfpath import RDFPath
|
|
14
|
+
from cognite.neat.rules.models.entities import (
|
|
15
|
+
ClassEntity,
|
|
16
|
+
Entity,
|
|
17
|
+
ReferenceEntity,
|
|
18
|
+
)
|
|
19
|
+
from cognite.neat.rules.models.information import InformationProperty
|
|
20
|
+
from cognite.neat.utils.utils import get_inheritance_path
|
|
8
21
|
|
|
22
|
+
T_Rules = TypeVar("T_Rules", bound=BaseRules)
|
|
23
|
+
T_Property = TypeVar("T_Property", bound=BaseModel)
|
|
24
|
+
T_Class = TypeVar("T_Class", bound=BaseModel)
|
|
25
|
+
T_ClassEntity = TypeVar("T_ClassEntity", bound=Entity)
|
|
26
|
+
T_PropertyEntity = TypeVar("T_PropertyEntity", bound=Entity | str)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Linkage(Generic[T_ClassEntity, T_PropertyEntity]):
|
|
31
|
+
source_class: T_ClassEntity
|
|
32
|
+
connecting_property: T_PropertyEntity
|
|
33
|
+
target_class: T_ClassEntity
|
|
34
|
+
max_occurrence: int | float | None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class LinkageSet(set, Generic[T_ClassEntity, T_PropertyEntity], Set[Linkage[T_ClassEntity, T_PropertyEntity]]):
|
|
38
|
+
@property
|
|
39
|
+
def source_class(self) -> set[T_ClassEntity]:
|
|
40
|
+
return {link.source_class for link in self}
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def target_class(self) -> set[T_ClassEntity]:
|
|
44
|
+
return {link.target_class for link in self}
|
|
45
|
+
|
|
46
|
+
def get_target_classes_by_source(self) -> dict[T_ClassEntity, set[T_ClassEntity]]:
|
|
47
|
+
target_classes_by_source: dict[T_ClassEntity, set[T_ClassEntity]] = defaultdict(set)
|
|
48
|
+
for link in self:
|
|
49
|
+
target_classes_by_source[link.source_class].add(link.target_class)
|
|
50
|
+
return target_classes_by_source
|
|
51
|
+
|
|
52
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
53
|
+
# Todo: Remove this method
|
|
54
|
+
return pd.DataFrame(
|
|
55
|
+
[
|
|
56
|
+
{
|
|
57
|
+
"source_class": link.source_class,
|
|
58
|
+
"connecting_property": link.connecting_property,
|
|
59
|
+
"target_class": link.target_class,
|
|
60
|
+
"max_occurrence": link.max_occurrence,
|
|
61
|
+
}
|
|
62
|
+
for link in self
|
|
63
|
+
]
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_PropertyEntity]):
|
|
68
|
+
def __init__(self, rules: T_Rules) -> None:
|
|
69
|
+
self.rules = rules
|
|
70
|
+
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def _get_classes(self) -> list[T_Class]:
|
|
73
|
+
raise NotImplementedError
|
|
74
|
+
|
|
75
|
+
@abstractmethod
|
|
76
|
+
def _get_properties(self) -> list[T_Property]:
|
|
77
|
+
raise NotImplementedError
|
|
78
|
+
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def _get_reference(self, class_or_property: T_Class | T_Property) -> ReferenceEntity | None:
|
|
81
|
+
raise NotImplementedError
|
|
82
|
+
|
|
83
|
+
@abstractmethod
|
|
84
|
+
def _get_cls_entity(self, class_: T_Class | T_Property) -> T_ClassEntity:
|
|
85
|
+
raise NotImplementedError
|
|
86
|
+
|
|
87
|
+
@abstractmethod
|
|
88
|
+
def _get_prop_entity(self, property_: T_Property) -> T_PropertyEntity:
|
|
89
|
+
raise NotImplementedError
|
|
90
|
+
|
|
91
|
+
@abstractmethod
|
|
92
|
+
def _get_cls_parents(self, class_: T_Class) -> list[T_ClassEntity] | None:
|
|
93
|
+
raise NotImplementedError
|
|
94
|
+
|
|
95
|
+
@abstractmethod
|
|
96
|
+
def _get_reference_rules(self) -> T_Rules | None:
|
|
97
|
+
raise NotImplementedError
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
@abstractmethod
|
|
101
|
+
def _set_cls_entity(cls, property_: T_Property, class_: T_ClassEntity) -> None:
|
|
102
|
+
raise NotImplementedError
|
|
103
|
+
|
|
104
|
+
@abstractmethod
|
|
105
|
+
def _get_object(self, property_: T_Property) -> T_ClassEntity | None:
|
|
106
|
+
raise NotImplementedError
|
|
107
|
+
|
|
108
|
+
@abstractmethod
|
|
109
|
+
def _get_max_occurrence(self, property_: T_Property) -> int | float | None:
|
|
110
|
+
raise NotImplementedError
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def directly_referred_classes(self) -> set[ClassEntity]:
|
|
114
|
+
ref_rules = self._get_reference_rules()
|
|
115
|
+
if ref_rules is None:
|
|
116
|
+
return set()
|
|
117
|
+
prefix = ref_rules.metadata.get_prefix()
|
|
118
|
+
return {
|
|
119
|
+
ref.as_class_entity()
|
|
120
|
+
for class_ in self._get_classes()
|
|
121
|
+
if isinstance((ref := self._get_reference(class_)), ReferenceEntity) and ref.prefix == prefix
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def inherited_referred_classes(self) -> set[ClassEntity]:
|
|
126
|
+
dir_referred_classes = self.directly_referred_classes
|
|
127
|
+
inherited_referred_classes = []
|
|
128
|
+
for class_ in dir_referred_classes:
|
|
129
|
+
inherited_referred_classes.extend(self.class_inheritance_path(class_))
|
|
130
|
+
return set(inherited_referred_classes)
|
|
131
|
+
|
|
132
|
+
# Todo Lru cache this method.
|
|
133
|
+
def class_parent_pairs(self) -> dict[T_ClassEntity, list[T_ClassEntity]]:
|
|
134
|
+
"""This only returns class - parent pairs only if parent is in the same data model"""
|
|
135
|
+
class_subclass_pairs: dict[T_ClassEntity, list[T_ClassEntity]] = {}
|
|
136
|
+
for cls_ in self._get_classes():
|
|
137
|
+
entity = self._get_cls_entity(cls_)
|
|
138
|
+
class_subclass_pairs[entity] = []
|
|
139
|
+
for parent in self._get_cls_parents(cls_) or []:
|
|
140
|
+
if parent.prefix == entity.prefix:
|
|
141
|
+
class_subclass_pairs[entity].append(parent)
|
|
142
|
+
else:
|
|
143
|
+
warnings.warn(
|
|
144
|
+
f"Parent class {parent} of class {cls_} is not in the same namespace, skipping !",
|
|
145
|
+
stacklevel=2,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
return class_subclass_pairs
|
|
149
|
+
|
|
150
|
+
def classes_with_properties(self, consider_inheritance: bool = False) -> dict[T_ClassEntity, list[T_Property]]:
|
|
151
|
+
"""Returns classes that have been defined in the data model.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
Dictionary of classes with a list of properties defined for them
|
|
158
|
+
|
|
159
|
+
!!! note "consider_inheritance"
|
|
160
|
+
If consider_inheritance is True, properties from parent classes will also be considered.
|
|
161
|
+
This means if a class has a parent class, and the parent class has properties defined for it,
|
|
162
|
+
while we do not have any properties defined for the child class, we will still consider the
|
|
163
|
+
properties from the parent class. If consider_inheritance is False, we will only consider
|
|
164
|
+
properties defined for the child class, thus if no properties are defined for the child class,
|
|
165
|
+
it will not be included in the returned dictionary.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
class_property_pairs: dict[T_ClassEntity, list[T_Property]] = defaultdict(list)
|
|
169
|
+
|
|
170
|
+
for property_ in self._get_properties():
|
|
171
|
+
class_property_pairs[self._get_cls_entity(property_)].append(property_) # type: ignore
|
|
172
|
+
|
|
173
|
+
if consider_inheritance:
|
|
174
|
+
class_parent_pairs = self.class_parent_pairs()
|
|
175
|
+
for class_ in class_parent_pairs:
|
|
176
|
+
self._add_inherited_properties(class_, class_property_pairs, class_parent_pairs)
|
|
177
|
+
|
|
178
|
+
return class_property_pairs
|
|
179
|
+
|
|
180
|
+
def class_inheritance_path(self, class_: ClassEntity) -> list[ClassEntity]:
|
|
181
|
+
class_parent_pairs = self.class_parent_pairs()
|
|
182
|
+
return get_inheritance_path(class_, class_parent_pairs)
|
|
183
|
+
|
|
184
|
+
@classmethod
|
|
185
|
+
def _add_inherited_properties(
|
|
186
|
+
cls,
|
|
187
|
+
class_: T_ClassEntity,
|
|
188
|
+
class_property_pairs: dict[T_ClassEntity, list[T_Property]],
|
|
189
|
+
class_parent_pairs: dict[T_ClassEntity, list[T_ClassEntity]],
|
|
190
|
+
):
|
|
191
|
+
inheritance_path = get_inheritance_path(class_, class_parent_pairs)
|
|
192
|
+
for parent in inheritance_path:
|
|
193
|
+
# ParentClassEntity -> ClassEntity to match the type of class_property_pairs
|
|
194
|
+
if parent in class_property_pairs:
|
|
195
|
+
for property_ in class_property_pairs[parent]:
|
|
196
|
+
property_ = property_.model_copy()
|
|
197
|
+
|
|
198
|
+
# This corresponds to importing properties from parent class
|
|
199
|
+
# making sure that the property is attached to desired child class
|
|
200
|
+
cls._set_cls_entity(property_, class_)
|
|
201
|
+
|
|
202
|
+
# need same if we have RDF path to make sure that the starting class is the
|
|
203
|
+
if class_ in class_property_pairs:
|
|
204
|
+
class_property_pairs[class_].append(property_)
|
|
205
|
+
else:
|
|
206
|
+
class_property_pairs[class_] = [property_]
|
|
207
|
+
|
|
208
|
+
def class_property_pairs(
|
|
209
|
+
self, only_rdfpath: bool = False, consider_inheritance: bool = False
|
|
210
|
+
) -> dict[T_ClassEntity, dict[T_PropertyEntity, T_Property]]:
|
|
211
|
+
"""Returns a dictionary of classes with a dictionary of properties associated with them.
|
|
212
|
+
|
|
213
|
+
Args:
|
|
214
|
+
only_rdfpath : To consider only properties which have rule `rdfpath` set. Defaults False
|
|
215
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
Dictionary of classes with a dictionary of properties associated with them.
|
|
219
|
+
|
|
220
|
+
!!! note "difference to get_classes_with_properties"
|
|
221
|
+
This method returns a dictionary of classes with a dictionary of properties associated with them.
|
|
222
|
+
While get_classes_with_properties returns a dictionary of classes with a list of
|
|
223
|
+
properties defined for them,
|
|
224
|
+
here we filter the properties based on the `only_rdfpath` parameter and only consider
|
|
225
|
+
the first definition of a property if it is defined more than once.
|
|
226
|
+
|
|
227
|
+
!!! note "only_rdfpath"
|
|
228
|
+
If only_rdfpath is True, only properties with RuleType.rdfpath will be returned as
|
|
229
|
+
a part of the dictionary of properties related to a class. Otherwise, all properties
|
|
230
|
+
will be returned.
|
|
231
|
+
|
|
232
|
+
!!! note "consider_inheritance"
|
|
233
|
+
If consider_inheritance is True, properties from parent classes will also be considered.
|
|
234
|
+
This means if a class has a parent class, and the parent class has properties defined for it,
|
|
235
|
+
while we do not have any properties defined for the child class, we will still consider the
|
|
236
|
+
properties from the parent class. If consider_inheritance is False, we will only consider
|
|
237
|
+
properties defined for the child class, thus if no properties are defined for the child class,
|
|
238
|
+
it will not be included in the returned dictionary.
|
|
239
|
+
"""
|
|
240
|
+
# TODO: https://cognitedata.atlassian.net/jira/software/projects/NEAT/boards/893?selectedIssue=NEAT-78
|
|
241
|
+
|
|
242
|
+
class_property_pairs: dict[T_ClassEntity, dict[T_PropertyEntity, T_Property]] = {}
|
|
243
|
+
|
|
244
|
+
for class_, properties in self.classes_with_properties(consider_inheritance).items():
|
|
245
|
+
processed_properties: dict[T_PropertyEntity, T_Property] = {}
|
|
246
|
+
for property_ in properties:
|
|
247
|
+
prop_entity = self._get_prop_entity(property_)
|
|
248
|
+
if prop_entity in processed_properties:
|
|
249
|
+
# TODO: use appropriate Warning class from _exceptions.py
|
|
250
|
+
# if missing make one !
|
|
251
|
+
warnings.warn(
|
|
252
|
+
f"Property {processed_properties} for {class_} has been defined more than once!"
|
|
253
|
+
" Only the first definition will be considered, skipping the rest..",
|
|
254
|
+
stacklevel=2,
|
|
255
|
+
)
|
|
256
|
+
continue
|
|
257
|
+
|
|
258
|
+
if (
|
|
259
|
+
only_rdfpath
|
|
260
|
+
and isinstance(property_, InformationProperty)
|
|
261
|
+
and isinstance(property_.transformation, RDFPath)
|
|
262
|
+
) or not only_rdfpath:
|
|
263
|
+
processed_properties[prop_entity] = property_
|
|
264
|
+
class_property_pairs[class_] = processed_properties
|
|
265
|
+
|
|
266
|
+
return class_property_pairs
|
|
267
|
+
|
|
268
|
+
def class_linkage(self, consider_inheritance: bool = False) -> LinkageSet[T_ClassEntity, T_PropertyEntity]:
|
|
269
|
+
"""Returns a set of class linkages in the data model.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
|
|
276
|
+
"""
|
|
277
|
+
class_linkage = LinkageSet[T_ClassEntity, T_PropertyEntity]()
|
|
278
|
+
|
|
279
|
+
class_property_pairs = self.classes_with_properties(consider_inheritance)
|
|
280
|
+
properties = list(itertools.chain.from_iterable(class_property_pairs.values()))
|
|
281
|
+
|
|
282
|
+
for property_ in properties:
|
|
283
|
+
object_ = self._get_object(property_)
|
|
284
|
+
if object_ is not None:
|
|
285
|
+
class_linkage.add(
|
|
286
|
+
Linkage(
|
|
287
|
+
source_class=self._get_cls_entity(property_),
|
|
288
|
+
connecting_property=self._get_prop_entity(property_),
|
|
289
|
+
target_class=object_,
|
|
290
|
+
max_occurrence=self._get_max_occurrence(property_),
|
|
291
|
+
)
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
return class_linkage
|
|
295
|
+
|
|
296
|
+
def connected_classes(self, consider_inheritance: bool = False) -> set[T_ClassEntity]:
|
|
297
|
+
"""Return a set of classes that are connected to other classes.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
301
|
+
|
|
302
|
+
Returns:
|
|
303
|
+
Set of classes that are connected to other classes
|
|
304
|
+
"""
|
|
305
|
+
class_linkage = self.class_linkage(consider_inheritance)
|
|
306
|
+
return class_linkage.source_class.union(class_linkage.target_class)
|
|
307
|
+
|
|
308
|
+
def defined_classes(self, consider_inheritance: bool = False) -> set[T_ClassEntity]:
|
|
309
|
+
"""Returns classes that have properties defined for them in the data model.
|
|
310
|
+
|
|
311
|
+
Args:
|
|
312
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
313
|
+
|
|
314
|
+
Returns:
|
|
315
|
+
Set of classes that have been defined in the data model
|
|
316
|
+
"""
|
|
317
|
+
class_property_pairs = self.classes_with_properties(consider_inheritance)
|
|
318
|
+
properties = list(itertools.chain.from_iterable(class_property_pairs.values()))
|
|
319
|
+
|
|
320
|
+
return {self._get_cls_entity(property) for property in properties}
|
|
321
|
+
|
|
322
|
+
def disconnected_classes(self, consider_inheritance: bool = False) -> set[T_ClassEntity]:
|
|
323
|
+
"""Return a set of classes that are disconnected (i.e. isolated) from other classes.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
327
|
+
|
|
328
|
+
Returns:
|
|
329
|
+
Set of classes that are disconnected from other classes
|
|
330
|
+
"""
|
|
331
|
+
return self.defined_classes(consider_inheritance) - self.connected_classes(consider_inheritance)
|
|
332
|
+
|
|
333
|
+
def symmetrically_connected_classes(
|
|
334
|
+
self, consider_inheritance: bool = False
|
|
335
|
+
) -> set[tuple[ClassEntity, ClassEntity]]:
|
|
336
|
+
"""Returns a set of pairs of symmetrically linked classes.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
340
|
+
|
|
341
|
+
Returns:
|
|
342
|
+
Set of pairs of symmetrically linked classes
|
|
343
|
+
|
|
344
|
+
!!! note "Symmetrically Connected Classes"
|
|
345
|
+
Symmetrically connected classes are classes that are connected to each other
|
|
346
|
+
in both directions. For example, if class A is connected to class B, and class B
|
|
347
|
+
is connected to class A, then classes A and B are symmetrically connected.
|
|
348
|
+
"""
|
|
349
|
+
|
|
350
|
+
# TODO: Find better name for this method
|
|
351
|
+
sym_pairs: set[tuple[ClassEntity, ClassEntity]] = set()
|
|
352
|
+
|
|
353
|
+
class_linkage = self.class_linkage(consider_inheritance)
|
|
354
|
+
if not class_linkage:
|
|
355
|
+
return sym_pairs
|
|
356
|
+
|
|
357
|
+
targets_by_source = class_linkage.get_target_classes_by_source()
|
|
358
|
+
for link in class_linkage:
|
|
359
|
+
source = link.source_class
|
|
360
|
+
target = link.target_class
|
|
361
|
+
|
|
362
|
+
if source in targets_by_source[source] and (source, target) not in sym_pairs:
|
|
363
|
+
sym_pairs.add((source, target))
|
|
364
|
+
return sym_pairs
|
|
365
|
+
|
|
366
|
+
def as_property_dict(
|
|
367
|
+
self,
|
|
368
|
+
) -> dict[T_PropertyEntity, list[T_Property]]:
|
|
369
|
+
"""This is used to capture all definitions of a property in the data model."""
|
|
370
|
+
property_dict: dict[T_PropertyEntity, list[T_Property]] = defaultdict(list)
|
|
371
|
+
for definition in self._get_properties():
|
|
372
|
+
property_dict[self._get_prop_entity(definition)].append(definition)
|
|
373
|
+
return property_dict
|
|
374
|
+
|
|
375
|
+
def as_class_dict(self) -> dict[str, T_Class]:
|
|
376
|
+
"""This is to simplify access to classes through dict."""
|
|
377
|
+
class_dict: dict[str, T_Class] = {}
|
|
378
|
+
for definition in self._get_classes():
|
|
379
|
+
entity = self._get_cls_entity(definition)
|
|
380
|
+
if entity.suffix in class_dict:
|
|
381
|
+
warnings.warn(
|
|
382
|
+
f"Class {entity} has been defined more than once! Only the first definition "
|
|
383
|
+
"will be considered, skipping the rest..",
|
|
384
|
+
stacklevel=2,
|
|
385
|
+
)
|
|
386
|
+
continue
|
|
387
|
+
class_dict[entity.suffix] = definition
|
|
388
|
+
return class_dict
|
|
9
389
|
|
|
10
|
-
class BaseAnalysis(ABC, Generic[T_Rules]):
|
|
11
390
|
@abstractmethod
|
|
12
|
-
def subset_rules(self, desired_classes: set[
|
|
13
|
-
raise NotImplementedError
|
|
391
|
+
def subset_rules(self, desired_classes: set[T_ClassEntity]) -> T_Rules:
|
|
392
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, cast
|
|
4
|
+
|
|
5
|
+
from pydantic import ValidationError
|
|
6
|
+
|
|
7
|
+
from cognite.neat.rules.models import SchemaCompleteness
|
|
8
|
+
from cognite.neat.rules.models._rdfpath import Hop, RDFPath, SingleProperty
|
|
9
|
+
from cognite.neat.rules.models.entities import ClassEntity, ReferenceEntity
|
|
10
|
+
from cognite.neat.rules.models.information import (
|
|
11
|
+
InformationClass,
|
|
12
|
+
InformationProperty,
|
|
13
|
+
InformationRules,
|
|
14
|
+
)
|
|
15
|
+
from cognite.neat.utils.utils import get_inheritance_path
|
|
16
|
+
|
|
17
|
+
from ._base import BaseAnalysis
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, InformationProperty, ClassEntity, str]):
|
|
21
|
+
"""Assumes analysis over only the complete schema"""
|
|
22
|
+
|
|
23
|
+
def _get_object(self, property_: InformationProperty) -> ClassEntity | None:
|
|
24
|
+
return property_.value_type if isinstance(property_.value_type, ClassEntity) else None
|
|
25
|
+
|
|
26
|
+
def _get_max_occurrence(self, property_: InformationProperty) -> int | float | None:
|
|
27
|
+
return property_.max_count
|
|
28
|
+
|
|
29
|
+
def _get_reference(self, class_or_property: InformationClass | InformationProperty) -> ReferenceEntity | None:
|
|
30
|
+
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
31
|
+
|
|
32
|
+
def _get_cls_entity(self, class_: InformationClass | InformationProperty) -> ClassEntity:
|
|
33
|
+
return class_.class_
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def _set_cls_entity(cls, property_: InformationProperty, class_: ClassEntity) -> None:
|
|
37
|
+
property_.class_ = class_
|
|
38
|
+
|
|
39
|
+
def _get_prop_entity(self, property_: InformationProperty) -> str:
|
|
40
|
+
return property_.property_
|
|
41
|
+
|
|
42
|
+
def _get_cls_parents(self, class_: InformationClass) -> list[ClassEntity] | None:
|
|
43
|
+
return list(class_.parent or []) or None
|
|
44
|
+
|
|
45
|
+
def _get_reference_rules(self) -> InformationRules | None:
|
|
46
|
+
return self.rules.reference
|
|
47
|
+
|
|
48
|
+
def _get_properties(self) -> list[InformationProperty]:
|
|
49
|
+
return list(self.rules.properties)
|
|
50
|
+
|
|
51
|
+
def _get_classes(self) -> list[InformationClass]:
|
|
52
|
+
return list(self.rules.classes)
|
|
53
|
+
|
|
54
|
+
def has_hop_transformations(self):
|
|
55
|
+
return any(
|
|
56
|
+
prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def define_property_renaming_config(self, class_: ClassEntity) -> dict[str, str]:
|
|
60
|
+
property_renaming_configuration = {}
|
|
61
|
+
|
|
62
|
+
if definitions := self.class_property_pairs(only_rdfpath=True, consider_inheritance=True).get(class_, None):
|
|
63
|
+
for property_id, definition in definitions.items():
|
|
64
|
+
if isinstance(
|
|
65
|
+
cast(RDFPath, definition.transformation).traversal,
|
|
66
|
+
SingleProperty,
|
|
67
|
+
):
|
|
68
|
+
graph_property = cast(
|
|
69
|
+
SingleProperty,
|
|
70
|
+
cast(RDFPath, definition.transformation).traversal,
|
|
71
|
+
).property.suffix
|
|
72
|
+
|
|
73
|
+
else:
|
|
74
|
+
graph_property = property_id
|
|
75
|
+
|
|
76
|
+
property_renaming_configuration[graph_property] = property_id
|
|
77
|
+
|
|
78
|
+
return property_renaming_configuration
|
|
79
|
+
|
|
80
|
+
def subset_rules(self, desired_classes: set[ClassEntity]) -> InformationRules:
|
|
81
|
+
"""
|
|
82
|
+
Subset rules to only include desired classes and their properties.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
desired_classes: Desired classes to include in the reduced data model
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Instance of InformationRules
|
|
89
|
+
|
|
90
|
+
!!! note "Inheritance"
|
|
91
|
+
If desired classes contain a class that is a subclass of another class(es), the parent class(es)
|
|
92
|
+
will be included in the reduced data model as well even though the parent class(es) are
|
|
93
|
+
not in the desired classes set. This is to ensure that the reduced data model is
|
|
94
|
+
consistent and complete.
|
|
95
|
+
|
|
96
|
+
!!! note "Partial Reduction"
|
|
97
|
+
This method does not perform checks if classes that are value types of desired classes
|
|
98
|
+
properties are part of desired classes. If a class is not part of desired classes, but it
|
|
99
|
+
is a value type of a property of a class that is part of desired classes, derived reduced
|
|
100
|
+
rules will be marked as partial.
|
|
101
|
+
|
|
102
|
+
!!! note "Validation"
|
|
103
|
+
This method will attempt to validate the reduced rules with custom validations.
|
|
104
|
+
If it fails, it will return a partial rules with a warning message, validated
|
|
105
|
+
only with base Pydantic validators.
|
|
106
|
+
"""
|
|
107
|
+
if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
|
|
108
|
+
raise ValueError("Rules are not complete cannot perform reduction!")
|
|
109
|
+
class_as_dict = self.as_class_dict()
|
|
110
|
+
class_parents_pairs = self.class_parent_pairs()
|
|
111
|
+
defined_classes = self.defined_classes(consider_inheritance=True)
|
|
112
|
+
|
|
113
|
+
possible_classes = defined_classes.intersection(desired_classes)
|
|
114
|
+
impossible_classes = desired_classes - possible_classes
|
|
115
|
+
|
|
116
|
+
# need to add all the parent classes of the desired classes to the possible classes
|
|
117
|
+
parents: set[ClassEntity] = set()
|
|
118
|
+
for class_ in possible_classes:
|
|
119
|
+
parents = parents.union({parent for parent in get_inheritance_path(class_, class_parents_pairs)})
|
|
120
|
+
possible_classes = possible_classes.union(parents)
|
|
121
|
+
|
|
122
|
+
if not possible_classes:
|
|
123
|
+
logging.error("None of the desired classes are defined in the data model!")
|
|
124
|
+
raise ValueError("None of the desired classes are defined in the data model!")
|
|
125
|
+
|
|
126
|
+
if impossible_classes:
|
|
127
|
+
logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
|
|
128
|
+
warnings.warn(
|
|
129
|
+
f"Could not find the following classes defined in the data model: {impossible_classes}",
|
|
130
|
+
stacklevel=2,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
reduced_data_model: dict[str, Any] = {
|
|
134
|
+
"metadata": self.rules.metadata.model_copy(),
|
|
135
|
+
"prefixes": (self.rules.prefixes or {}).copy(),
|
|
136
|
+
"classes": [],
|
|
137
|
+
"properties": [],
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
|
|
141
|
+
for class_ in possible_classes:
|
|
142
|
+
reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
|
|
143
|
+
|
|
144
|
+
class_property_pairs = self.classes_with_properties(consider_inheritance=False)
|
|
145
|
+
|
|
146
|
+
for class_, properties in class_property_pairs.items():
|
|
147
|
+
if class_ in possible_classes:
|
|
148
|
+
reduced_data_model["properties"].extend(properties)
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
return type(self.rules)(**reduced_data_model)
|
|
152
|
+
except ValidationError as e:
|
|
153
|
+
warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
|
|
154
|
+
reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
|
|
155
|
+
return type(self.rules).model_construct(**reduced_data_model)
|
|
@@ -10,7 +10,7 @@ from rdflib.collection import Collection as GraphCollection
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
12
|
from cognite.neat.rules import exceptions
|
|
13
|
-
from cognite.neat.rules.analysis import
|
|
13
|
+
from cognite.neat.rules.analysis import InformationAnalysis
|
|
14
14
|
from cognite.neat.rules.models import DMSRules
|
|
15
15
|
from cognite.neat.rules.models.data_types import DataType
|
|
16
16
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
@@ -109,11 +109,11 @@ class Ontology(OntologyModel):
|
|
|
109
109
|
if rules.metadata.namespace is None:
|
|
110
110
|
raise exceptions.MissingDataModelPrefixOrNamespace()
|
|
111
111
|
|
|
112
|
-
class_dict =
|
|
112
|
+
class_dict = InformationAnalysis(rules).as_class_dict()
|
|
113
113
|
return cls(
|
|
114
114
|
properties=[
|
|
115
115
|
OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
|
|
116
|
-
for definition in
|
|
116
|
+
for definition in InformationAnalysis(rules).as_property_dict().values()
|
|
117
117
|
],
|
|
118
118
|
classes=[
|
|
119
119
|
OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
|
|
@@ -125,7 +125,7 @@ class Ontology(OntologyModel):
|
|
|
125
125
|
list(properties.values()),
|
|
126
126
|
rules.metadata.namespace,
|
|
127
127
|
)
|
|
128
|
-
for class_, properties in
|
|
128
|
+
for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
|
|
129
129
|
]
|
|
130
130
|
+ [
|
|
131
131
|
SHACLNodeShape.from_rules(
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
|
-
from typing import cast
|
|
4
3
|
|
|
5
4
|
import cognite.neat.rules.issues.importing
|
|
6
5
|
from cognite.neat.rules import issues
|
|
@@ -22,7 +21,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
22
21
|
)
|
|
23
22
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
23
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
|
-
from cognite.neat.rules.models.entities import ClassEntity
|
|
24
|
+
from cognite.neat.rules.models.entities import ClassEntity
|
|
26
25
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
27
26
|
|
|
28
27
|
|
|
@@ -89,12 +88,7 @@ class _DTDLConverter:
|
|
|
89
88
|
name=item.display_name,
|
|
90
89
|
description=item.description,
|
|
91
90
|
comment=item.comment,
|
|
92
|
-
parent=[
|
|
93
|
-
cast(ParentClassEntity, parent_entity)
|
|
94
|
-
for parent in item.extends or []
|
|
95
|
-
if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
|
|
96
|
-
]
|
|
97
|
-
or None,
|
|
91
|
+
parent=[parent.as_class_id() for parent in item.extends or []] or None,
|
|
98
92
|
)
|
|
99
93
|
self.classes.append(class_)
|
|
100
94
|
for sub_item_or_id in item.contents or []:
|
|
@@ -7,7 +7,7 @@ from rdflib import Graph, Namespace, URIRef
|
|
|
7
7
|
from rdflib import Literal as RdfLiteral
|
|
8
8
|
|
|
9
9
|
import cognite.neat.rules.issues as issues
|
|
10
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
10
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
11
11
|
from cognite.neat.graph.stores import NeatGraphStore
|
|
12
12
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
13
13
|
from cognite.neat.rules.issues import IssueList
|
|
@@ -204,7 +204,7 @@ class InferenceImporter(BaseImporter):
|
|
|
204
204
|
"""
|
|
205
205
|
classes: dict[str, dict] = {}
|
|
206
206
|
properties: dict[str, dict] = {}
|
|
207
|
-
prefixes: dict[str, Namespace] =
|
|
207
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
208
208
|
|
|
209
209
|
query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
|
|
210
210
|
# Adds default namespace to prefixes
|
|
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
import math
|
|
8
8
|
import sys
|
|
9
9
|
import types
|
|
10
|
-
from abc import abstractmethod
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
11
|
from collections.abc import Callable, Iterator
|
|
12
12
|
from functools import wraps
|
|
13
13
|
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
|
|
@@ -18,7 +18,6 @@ from pydantic import (
|
|
|
18
18
|
BeforeValidator,
|
|
19
19
|
ConfigDict,
|
|
20
20
|
Field,
|
|
21
|
-
HttpUrl,
|
|
22
21
|
PlainSerializer,
|
|
23
22
|
constr,
|
|
24
23
|
field_validator,
|
|
@@ -219,10 +218,6 @@ class RuleModel(BaseModel):
|
|
|
219
218
|
return headers_by_sheet
|
|
220
219
|
|
|
221
220
|
|
|
222
|
-
class URL(BaseModel):
|
|
223
|
-
url: HttpUrl
|
|
224
|
-
|
|
225
|
-
|
|
226
221
|
class BaseMetadata(RuleModel):
|
|
227
222
|
"""
|
|
228
223
|
Metadata model for data model
|
|
@@ -252,8 +247,13 @@ class BaseMetadata(RuleModel):
|
|
|
252
247
|
"""Returns a unique identifier for the metadata."""
|
|
253
248
|
raise NotImplementedError()
|
|
254
249
|
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def get_prefix(self) -> str:
|
|
252
|
+
"""Returns the prefix for the metadata."""
|
|
253
|
+
raise NotImplementedError()
|
|
254
|
+
|
|
255
255
|
|
|
256
|
-
class BaseRules(RuleModel):
|
|
256
|
+
class BaseRules(RuleModel, ABC):
|
|
257
257
|
"""
|
|
258
258
|
Rules is a core concept in `neat`. This represents fusion of data model
|
|
259
259
|
definitions and (optionally) the transformation rules used to transform the data/graph
|