cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +1 -1
- cognite/neat/core/_data_model/exporters/__init__.py +5 -5
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +6 -6
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
- cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
- cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
- cognite/neat/core/_data_model/models/__init__.py +8 -8
- cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
- cognite/neat/core/_data_model/models/_base_verified.py +3 -3
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
- cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +27 -23
- cognite/neat/core/_data_model/transformers/_base.py +26 -19
- cognite/neat/core/_data_model/transformers/_converters.py +703 -618
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +63 -54
- cognite/neat/core/_instances/extractors/_base.py +1 -1
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
- cognite/neat/core/_instances/loaders/_base.py +2 -2
- cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +22 -20
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +8 -6
- cognite/neat/session/_read.py +2 -2
- cognite/neat/session/_set.py +3 -3
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +13 -13
- cognite/neat/session/_subset.py +12 -9
- cognite/neat/session/_template.py +13 -13
- cognite/neat/session/_to.py +17 -17
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -11,21 +11,21 @@ import pandas as pd
|
|
|
11
11
|
from cognite.client import data_modeling as dm
|
|
12
12
|
from rdflib import URIRef
|
|
13
13
|
|
|
14
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
14
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
15
15
|
from cognite.neat.core._data_model.models.conceptual import (
|
|
16
|
-
|
|
16
|
+
Concept,
|
|
17
17
|
ConceptualProperty,
|
|
18
18
|
)
|
|
19
|
-
from cognite.neat.core._data_model.models.dms import DMSProperty
|
|
20
|
-
from cognite.neat.core._data_model.models.dms._rules import DMSView
|
|
21
19
|
from cognite.neat.core._data_model.models.entities import (
|
|
22
|
-
|
|
20
|
+
ConceptEntity,
|
|
23
21
|
MultiValueTypeInfo,
|
|
24
22
|
ViewEntity,
|
|
25
23
|
)
|
|
26
24
|
from cognite.neat.core._data_model.models.entities._single_value import (
|
|
27
25
|
UnknownEntity,
|
|
28
26
|
)
|
|
27
|
+
from cognite.neat.core._data_model.models.physical import PhysicalProperty
|
|
28
|
+
from cognite.neat.core._data_model.models.physical._verified import PhysicalView
|
|
29
29
|
from cognite.neat.core._issues.errors import NeatValueError
|
|
30
30
|
from cognite.neat.core._issues.warnings import NeatValueWarning
|
|
31
31
|
|
|
@@ -34,35 +34,35 @@ T_Hashable = TypeVar("T_Hashable", bound=Hashable)
|
|
|
34
34
|
|
|
35
35
|
@dataclass(frozen=True)
|
|
36
36
|
class Linkage:
|
|
37
|
-
|
|
37
|
+
source_concept: ConceptEntity
|
|
38
38
|
connecting_property: str
|
|
39
|
-
|
|
39
|
+
target_concept: ConceptEntity
|
|
40
40
|
max_occurrence: int | float | None
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
class LinkageSet(set, Set[Linkage]):
|
|
44
44
|
@property
|
|
45
|
-
def
|
|
46
|
-
return {link.
|
|
45
|
+
def source_concept(self) -> set[ConceptEntity]:
|
|
46
|
+
return {link.source_concept for link in self}
|
|
47
47
|
|
|
48
48
|
@property
|
|
49
|
-
def
|
|
50
|
-
return {link.
|
|
49
|
+
def target_concept(self) -> set[ConceptEntity]:
|
|
50
|
+
return {link.target_concept for link in self}
|
|
51
51
|
|
|
52
|
-
def
|
|
53
|
-
|
|
52
|
+
def get_target_concepts_by_source(self) -> dict[ConceptEntity, set[ConceptEntity]]:
|
|
53
|
+
target_concepts_by_source: dict[ConceptEntity, set[ConceptEntity]] = defaultdict(set)
|
|
54
54
|
for link in self:
|
|
55
|
-
|
|
56
|
-
return
|
|
55
|
+
target_concepts_by_source[link.source_concept].add(link.target_concept)
|
|
56
|
+
return target_concepts_by_source
|
|
57
57
|
|
|
58
58
|
def to_pandas(self) -> pd.DataFrame:
|
|
59
59
|
# Todo: Remove this method
|
|
60
60
|
return pd.DataFrame(
|
|
61
61
|
[
|
|
62
62
|
{
|
|
63
|
-
"
|
|
63
|
+
"source_concept": link.source_concept,
|
|
64
64
|
"connecting_property": link.connecting_property,
|
|
65
|
-
"
|
|
65
|
+
"target_concept": link.target_concept,
|
|
66
66
|
"max_occurrence": link.max_occurrence,
|
|
67
67
|
}
|
|
68
68
|
for link in self
|
|
@@ -110,109 +110,112 @@ class ViewQueryDict(dict, MutableMapping[dm.ViewId, ViewQuery]):
|
|
|
110
110
|
return super().popitem()
|
|
111
111
|
|
|
112
112
|
|
|
113
|
-
class
|
|
113
|
+
class DataModelAnalysis:
|
|
114
114
|
def __init__(
|
|
115
115
|
self,
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
conceptual: ConceptualDataModel | None = None,
|
|
117
|
+
physical: PhysicalDataModel | None = None,
|
|
118
118
|
) -> None:
|
|
119
|
-
self.
|
|
120
|
-
self.
|
|
119
|
+
self._conceptual = conceptual
|
|
120
|
+
self._physical = physical
|
|
121
121
|
|
|
122
122
|
@property
|
|
123
|
-
def
|
|
124
|
-
if self.
|
|
125
|
-
raise NeatValueError("
|
|
126
|
-
return self.
|
|
123
|
+
def conceptual(self) -> ConceptualDataModel:
|
|
124
|
+
if self._conceptual is None:
|
|
125
|
+
raise NeatValueError("Conceptual Data Model is required for this analysis")
|
|
126
|
+
return self._conceptual
|
|
127
127
|
|
|
128
128
|
@property
|
|
129
|
-
def
|
|
130
|
-
if self.
|
|
131
|
-
raise NeatValueError("
|
|
132
|
-
return self.
|
|
129
|
+
def physical(self) -> PhysicalDataModel:
|
|
130
|
+
if self._physical is None:
|
|
131
|
+
raise NeatValueError("Physical Data Model is required for this analysis")
|
|
132
|
+
return self._physical
|
|
133
133
|
|
|
134
|
-
def
|
|
134
|
+
def parents_by_concept(
|
|
135
135
|
self, include_ancestors: bool = False, include_different_space: bool = False
|
|
136
|
-
) -> dict[
|
|
137
|
-
"""Get a dictionary of
|
|
136
|
+
) -> dict[ConceptEntity, set[ConceptEntity]]:
|
|
137
|
+
"""Get a dictionary of concepts and their parents.
|
|
138
138
|
|
|
139
139
|
Args:
|
|
140
140
|
include_ancestors (bool, optional): Include ancestors of the parents. Defaults to False.
|
|
141
141
|
include_different_space (bool, optional): Include parents from different spaces. Defaults to False.
|
|
142
142
|
|
|
143
143
|
Returns:
|
|
144
|
-
dict[
|
|
144
|
+
dict[ConceptEntity, set[ConceptEntity]]: Values parents with concept as key.
|
|
145
145
|
"""
|
|
146
|
-
|
|
147
|
-
for
|
|
148
|
-
|
|
149
|
-
for parent in
|
|
150
|
-
if include_different_space or parent.prefix ==
|
|
151
|
-
|
|
146
|
+
parents_by_concept: dict[ConceptEntity, set[ConceptEntity]] = {}
|
|
147
|
+
for concept in self.conceptual.concepts:
|
|
148
|
+
parents_by_concept[concept.concept] = set()
|
|
149
|
+
for parent in concept.implements or []:
|
|
150
|
+
if include_different_space or parent.prefix == concept.concept.prefix:
|
|
151
|
+
parents_by_concept[concept.concept].add(parent)
|
|
152
152
|
else:
|
|
153
153
|
warnings.warn(
|
|
154
154
|
NeatValueWarning(
|
|
155
|
-
f"Parent
|
|
155
|
+
f"Parent concept {parent} of concept {concept} is not in the same namespace, skipping!"
|
|
156
156
|
),
|
|
157
157
|
stacklevel=2,
|
|
158
158
|
)
|
|
159
159
|
if include_ancestors:
|
|
160
|
-
self._include_ancestors(
|
|
160
|
+
self._include_ancestors(parents_by_concept)
|
|
161
161
|
|
|
162
|
-
return
|
|
162
|
+
return parents_by_concept
|
|
163
163
|
|
|
164
164
|
@staticmethod
|
|
165
|
-
def _include_ancestors(
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
165
|
+
def _include_ancestors(
|
|
166
|
+
parents_by_concept: dict[T_Hashable, set[T_Hashable]],
|
|
167
|
+
) -> None:
|
|
168
|
+
# Topological sort to ensure that concepts include all ancestors
|
|
169
|
+
for concept_entity in list(TopologicalSorter(parents_by_concept).static_order()):
|
|
170
|
+
if concept_entity not in parents_by_concept:
|
|
169
171
|
continue
|
|
170
|
-
|
|
172
|
+
parents_by_concept[concept_entity] |= {
|
|
171
173
|
grand_parent
|
|
172
|
-
for parent in
|
|
173
|
-
for grand_parent in
|
|
174
|
+
for parent in parents_by_concept[concept_entity]
|
|
175
|
+
for grand_parent in parents_by_concept.get(parent, set())
|
|
174
176
|
}
|
|
175
177
|
|
|
176
|
-
def
|
|
178
|
+
def properties_by_concepts(
|
|
177
179
|
self, include_ancestors: bool = False, include_different_space: bool = False
|
|
178
|
-
) -> dict[
|
|
179
|
-
"""Get a dictionary of
|
|
180
|
+
) -> dict[ConceptEntity, list[ConceptualProperty]]:
|
|
181
|
+
"""Get a dictionary of concepts and their properties.
|
|
180
182
|
|
|
181
183
|
Args:
|
|
182
|
-
include_ancestors: Whether to include properties from parent
|
|
183
|
-
include_different_space: Whether to include properties from parent
|
|
184
|
+
include_ancestors: Whether to include properties from parent concepts.
|
|
185
|
+
include_different_space: Whether to include properties from parent concepts in different spaces.
|
|
184
186
|
|
|
185
187
|
Returns:
|
|
186
|
-
dict[
|
|
188
|
+
dict[ConceptEntity, list[ConceptualProperty]]: Values properties with concept as key.
|
|
187
189
|
|
|
188
190
|
"""
|
|
189
|
-
|
|
190
|
-
for prop in self.
|
|
191
|
-
|
|
191
|
+
properties_by_concepts: dict[ConceptEntity, list[ConceptualProperty]] = defaultdict(list)
|
|
192
|
+
for prop in self.conceptual.properties:
|
|
193
|
+
properties_by_concepts[prop.concept].append(prop)
|
|
192
194
|
|
|
193
195
|
if include_ancestors:
|
|
194
|
-
|
|
195
|
-
include_ancestors=include_ancestors,
|
|
196
|
+
parents_by_concepts = self.parents_by_concept(
|
|
197
|
+
include_ancestors=include_ancestors,
|
|
198
|
+
include_different_space=include_different_space,
|
|
196
199
|
)
|
|
197
|
-
for
|
|
198
|
-
|
|
200
|
+
for concept, parents in parents_by_concepts.items():
|
|
201
|
+
concept_properties = {prop.property_ for prop in properties_by_concepts[concept]}
|
|
199
202
|
for parent in parents:
|
|
200
|
-
for parent_prop in
|
|
201
|
-
if parent_prop.property_ not in
|
|
202
|
-
child_prop = parent_prop.model_copy(update={"
|
|
203
|
-
|
|
204
|
-
|
|
203
|
+
for parent_prop in properties_by_concepts[parent]:
|
|
204
|
+
if parent_prop.property_ not in concept_properties:
|
|
205
|
+
child_prop = parent_prop.model_copy(update={"concept": concept})
|
|
206
|
+
properties_by_concepts[concept].append(child_prop)
|
|
207
|
+
concept_properties.add(child_prop.property_)
|
|
205
208
|
|
|
206
|
-
return
|
|
209
|
+
return properties_by_concepts
|
|
207
210
|
|
|
208
211
|
def implements_by_view(
|
|
209
212
|
self, include_ancestors: bool = False, include_different_space: bool = False
|
|
210
213
|
) -> dict[ViewEntity, set[ViewEntity]]:
|
|
211
214
|
"""Get a dictionary of views and their implemented views."""
|
|
212
|
-
# This is a duplicate fo the
|
|
215
|
+
# This is a duplicate fo the parent_by_concept method, but for views
|
|
213
216
|
# The choice to duplicate the code is to avoid generics which will make the code less readable
|
|
214
217
|
implements_by_view: dict[ViewEntity, set[ViewEntity]] = {}
|
|
215
|
-
for view in self.
|
|
218
|
+
for view in self.physical.views:
|
|
216
219
|
implements_by_view[view.view] = set()
|
|
217
220
|
for implements in view.implements or []:
|
|
218
221
|
if include_different_space or implements.space == view.view.space:
|
|
@@ -230,12 +233,12 @@ class RulesAnalysis:
|
|
|
230
233
|
|
|
231
234
|
def properties_by_view(
|
|
232
235
|
self, include_ancestors: bool = False, include_different_space: bool = False
|
|
233
|
-
) -> dict[ViewEntity, list[
|
|
236
|
+
) -> dict[ViewEntity, list[PhysicalProperty]]:
|
|
234
237
|
"""Get a dictionary of views and their properties."""
|
|
235
|
-
# This is a duplicate fo the
|
|
238
|
+
# This is a duplicate fo the properties_by_concept method, but for views
|
|
236
239
|
# The choice to duplicate the code is to avoid generics which will make the code less readable.
|
|
237
|
-
properties_by_views: dict[ViewEntity, list[
|
|
238
|
-
for prop in self.
|
|
240
|
+
properties_by_views: dict[ViewEntity, list[PhysicalProperty]] = defaultdict(list)
|
|
241
|
+
for prop in self.physical.properties:
|
|
239
242
|
properties_by_views[prop.view].append(prop)
|
|
240
243
|
|
|
241
244
|
if include_ancestors:
|
|
@@ -254,11 +257,11 @@ class RulesAnalysis:
|
|
|
254
257
|
return properties_by_views
|
|
255
258
|
|
|
256
259
|
@property
|
|
257
|
-
def
|
|
260
|
+
def conceptual_uri_by_view(self) -> dict[ViewEntity, URIRef]:
|
|
258
261
|
"""Get the logical URI by view."""
|
|
259
|
-
return {view.view: view.
|
|
262
|
+
return {view.view: view.conceptual for view in self.physical.views if view.conceptual}
|
|
260
263
|
|
|
261
|
-
def
|
|
264
|
+
def conceptual_uri_by_property_by_view(
|
|
262
265
|
self,
|
|
263
266
|
include_ancestors: bool = False,
|
|
264
267
|
include_different_space: bool = False,
|
|
@@ -267,68 +270,68 @@ class RulesAnalysis:
|
|
|
267
270
|
properties_by_view = self.properties_by_view(include_ancestors, include_different_space)
|
|
268
271
|
|
|
269
272
|
return {
|
|
270
|
-
view: {prop.view_property: prop.
|
|
273
|
+
view: {prop.view_property: prop.conceptual for prop in properties if prop.conceptual}
|
|
271
274
|
for view, properties in properties_by_view.items()
|
|
272
275
|
}
|
|
273
276
|
|
|
274
277
|
@property
|
|
275
|
-
def
|
|
276
|
-
"""Get a dictionary of
|
|
277
|
-
|
|
278
|
+
def _concept_by_neat_id(self) -> dict[URIRef, Concept]:
|
|
279
|
+
"""Get a dictionary of concept neat IDs to
|
|
280
|
+
concept entities."""
|
|
278
281
|
|
|
279
|
-
return {cls.neatId: cls for cls in self.
|
|
282
|
+
return {cls.neatId: cls for cls in self.conceptual.concepts if cls.neatId}
|
|
280
283
|
|
|
281
|
-
def
|
|
282
|
-
"""Get a dictionary of
|
|
284
|
+
def concept_by_suffix(self) -> dict[str, Concept]:
|
|
285
|
+
"""Get a dictionary of concept suffixes to concept entities."""
|
|
283
286
|
# TODO: Remove this method
|
|
284
|
-
|
|
285
|
-
for definition in self.
|
|
286
|
-
entity = definition.
|
|
287
|
-
if entity.suffix in
|
|
287
|
+
concept_dict: dict[str, Concept] = {}
|
|
288
|
+
for definition in self.conceptual.concepts:
|
|
289
|
+
entity = definition.concept
|
|
290
|
+
if entity.suffix in concept_dict:
|
|
288
291
|
warnings.warn(
|
|
289
292
|
NeatValueWarning(
|
|
290
|
-
f"
|
|
293
|
+
f"Concept {entity} has been defined more than once! Only the first definition "
|
|
291
294
|
"will be considered, skipping the rest.."
|
|
292
295
|
),
|
|
293
296
|
stacklevel=2,
|
|
294
297
|
)
|
|
295
298
|
continue
|
|
296
|
-
|
|
297
|
-
return
|
|
299
|
+
concept_dict[entity.suffix] = definition
|
|
300
|
+
return concept_dict
|
|
298
301
|
|
|
299
302
|
@property
|
|
300
|
-
def
|
|
301
|
-
"""Get a dictionary of
|
|
302
|
-
|
|
303
|
-
return {cls.
|
|
303
|
+
def concept_by_concept_entity(self) -> dict[ConceptEntity, Concept]:
|
|
304
|
+
"""Get a dictionary of concept entities to concept entities."""
|
|
305
|
+
data_model = self.conceptual
|
|
306
|
+
return {cls.concept: cls for cls in data_model.concepts}
|
|
304
307
|
|
|
305
308
|
@property
|
|
306
|
-
def view_by_view_entity(self) -> dict[ViewEntity,
|
|
307
|
-
"""Get a dictionary of
|
|
308
|
-
|
|
309
|
-
return {view.view: view for view in
|
|
309
|
+
def view_by_view_entity(self) -> dict[ViewEntity, PhysicalView]:
|
|
310
|
+
"""Get a dictionary of views to view entities."""
|
|
311
|
+
data_model = self.physical
|
|
312
|
+
return {view.view: view for view in data_model.views}
|
|
310
313
|
|
|
311
314
|
def property_by_id(self) -> dict[str, list[ConceptualProperty]]:
|
|
312
315
|
"""Get a dictionary of property IDs to property entities."""
|
|
313
316
|
property_dict: dict[str, list[ConceptualProperty]] = defaultdict(list)
|
|
314
|
-
for prop in self.
|
|
317
|
+
for prop in self.conceptual.properties:
|
|
315
318
|
property_dict[prop.property_].append(prop)
|
|
316
319
|
return property_dict
|
|
317
320
|
|
|
318
|
-
def
|
|
321
|
+
def properties_by_id_by_concept(
|
|
319
322
|
self,
|
|
320
323
|
has_instance_source: bool = False,
|
|
321
324
|
include_ancestors: bool = False,
|
|
322
|
-
) -> dict[
|
|
323
|
-
"""Get a dictionary of
|
|
324
|
-
|
|
325
|
-
for
|
|
325
|
+
) -> dict[ConceptEntity, dict[str, ConceptualProperty]]:
|
|
326
|
+
"""Get a dictionary of concept entities to dictionaries of property IDs to property entities."""
|
|
327
|
+
concept_property_pairs: dict[ConceptEntity, dict[str, ConceptualProperty]] = {}
|
|
328
|
+
for concept, properties in self.properties_by_concepts(include_ancestors).items():
|
|
326
329
|
processed_properties: dict[str, ConceptualProperty] = {}
|
|
327
330
|
for prop in properties:
|
|
328
331
|
if prop.property_ in processed_properties:
|
|
329
332
|
warnings.warn(
|
|
330
333
|
NeatValueWarning(
|
|
331
|
-
f"Property {processed_properties} for {
|
|
334
|
+
f"Property {processed_properties} for {concept} has been defined more than once!"
|
|
332
335
|
" Only the first definition will be considered, skipping the rest.."
|
|
333
336
|
),
|
|
334
337
|
stacklevel=2,
|
|
@@ -337,34 +340,34 @@ class RulesAnalysis:
|
|
|
337
340
|
if has_instance_source and prop.instance_source is None:
|
|
338
341
|
continue
|
|
339
342
|
processed_properties[prop.property_] = prop
|
|
340
|
-
|
|
343
|
+
concept_property_pairs[concept] = processed_properties
|
|
341
344
|
|
|
342
|
-
return
|
|
345
|
+
return concept_property_pairs
|
|
343
346
|
|
|
344
347
|
def defined_views(self, include_ancestors: bool = False) -> set[ViewEntity]:
|
|
345
348
|
properties_by_view = self.properties_by_view(include_ancestors)
|
|
346
349
|
return {prop.view for prop in itertools.chain.from_iterable(properties_by_view.values())}
|
|
347
350
|
|
|
348
|
-
def
|
|
351
|
+
def defined_concepts(
|
|
349
352
|
self,
|
|
350
353
|
include_ancestors: bool = False,
|
|
351
|
-
) -> set[
|
|
352
|
-
"""Returns
|
|
354
|
+
) -> set[ConceptEntity]:
|
|
355
|
+
"""Returns concepts that have properties defined for them in the data model.
|
|
353
356
|
|
|
354
357
|
Args:
|
|
355
358
|
include_ancestors: Whether to consider inheritance or not. Defaults False
|
|
356
359
|
|
|
357
360
|
Returns:
|
|
358
|
-
Set of
|
|
361
|
+
Set of concepts that have been defined in the data model
|
|
359
362
|
"""
|
|
360
|
-
|
|
361
|
-
return {prop.
|
|
363
|
+
properties_by_concept = self.properties_by_concepts(include_ancestors)
|
|
364
|
+
return {prop.concept for prop in itertools.chain.from_iterable(properties_by_concept.values())}
|
|
362
365
|
|
|
363
|
-
def
|
|
366
|
+
def concept_linkage(
|
|
364
367
|
self,
|
|
365
368
|
include_ancestors: bool = False,
|
|
366
369
|
) -> LinkageSet:
|
|
367
|
-
"""Returns a set of
|
|
370
|
+
"""Returns a set of concept linkages in the data model.
|
|
368
371
|
|
|
369
372
|
Args:
|
|
370
373
|
include_ancestors: Whether to consider inheritance or not. Defaults False
|
|
@@ -372,51 +375,51 @@ class RulesAnalysis:
|
|
|
372
375
|
Returns:
|
|
373
376
|
|
|
374
377
|
"""
|
|
375
|
-
|
|
378
|
+
concept_linkage = LinkageSet()
|
|
376
379
|
|
|
377
|
-
|
|
380
|
+
properties_by_concept = self.properties_by_concepts(include_ancestors)
|
|
378
381
|
|
|
379
382
|
prop: ConceptualProperty
|
|
380
|
-
for prop in itertools.chain.from_iterable(
|
|
381
|
-
if not isinstance(prop.value_type,
|
|
383
|
+
for prop in itertools.chain.from_iterable(properties_by_concept.values()):
|
|
384
|
+
if not isinstance(prop.value_type, ConceptEntity):
|
|
382
385
|
continue
|
|
383
|
-
|
|
386
|
+
concept_linkage.add(
|
|
384
387
|
Linkage(
|
|
385
|
-
|
|
388
|
+
source_concept=prop.concept,
|
|
386
389
|
connecting_property=prop.property_,
|
|
387
|
-
|
|
390
|
+
target_concept=prop.value_type,
|
|
388
391
|
max_occurrence=prop.max_count,
|
|
389
392
|
)
|
|
390
393
|
)
|
|
391
394
|
|
|
392
|
-
return
|
|
395
|
+
return concept_linkage
|
|
393
396
|
|
|
394
|
-
def
|
|
397
|
+
def symmetrically_connected_concepts(
|
|
395
398
|
self,
|
|
396
399
|
include_ancestors: bool = False,
|
|
397
|
-
) -> set[tuple[
|
|
398
|
-
"""Returns a set of pairs of symmetrically linked
|
|
400
|
+
) -> set[tuple[ConceptEntity, ConceptEntity]]:
|
|
401
|
+
"""Returns a set of pairs of symmetrically linked concepts.
|
|
399
402
|
|
|
400
403
|
Args:
|
|
401
404
|
include_ancestors: Whether to consider inheritance or not. Defaults False
|
|
402
405
|
|
|
403
406
|
Returns:
|
|
404
|
-
Set of pairs of symmetrically linked
|
|
407
|
+
Set of pairs of symmetrically linked concepts
|
|
405
408
|
|
|
406
|
-
!!! note "Symmetrically Connected
|
|
407
|
-
Symmetrically connected
|
|
408
|
-
in both directions. For example, if
|
|
409
|
-
is connected to
|
|
409
|
+
!!! note "Symmetrically Connected Concepts"
|
|
410
|
+
Symmetrically connected concepts are concepts that are connected to each other
|
|
411
|
+
in both directions. For example, if concept A is connected to concept B, and concept B
|
|
412
|
+
is connected to concept A, then concepts A and B are symmetrically connected.
|
|
410
413
|
"""
|
|
411
|
-
sym_pairs: set[tuple[
|
|
412
|
-
|
|
413
|
-
if not
|
|
414
|
+
sym_pairs: set[tuple[ConceptEntity, ConceptEntity]] = set()
|
|
415
|
+
concept_linkage = self.concept_linkage(include_ancestors)
|
|
416
|
+
if not concept_linkage:
|
|
414
417
|
return sym_pairs
|
|
415
418
|
|
|
416
|
-
targets_by_source =
|
|
417
|
-
for link in
|
|
418
|
-
source = link.
|
|
419
|
-
target = link.
|
|
419
|
+
targets_by_source = concept_linkage.get_target_concepts_by_source()
|
|
420
|
+
for link in concept_linkage:
|
|
421
|
+
source = link.source_concept
|
|
422
|
+
target = link.target_concept
|
|
420
423
|
|
|
421
424
|
if source in targets_by_source[source] and (source, target) not in sym_pairs:
|
|
422
425
|
sym_pairs.add((source, target))
|
|
@@ -426,51 +429,51 @@ class RulesAnalysis:
|
|
|
426
429
|
def _properties_by_neat_id(self, format: Literal["info"] = "info") -> dict[URIRef, ConceptualProperty]: ...
|
|
427
430
|
|
|
428
431
|
@overload
|
|
429
|
-
def _properties_by_neat_id(self, format: Literal["dms"] = "dms") -> dict[URIRef,
|
|
432
|
+
def _properties_by_neat_id(self, format: Literal["dms"] = "dms") -> dict[URIRef, PhysicalProperty]: ...
|
|
430
433
|
|
|
431
434
|
def _properties_by_neat_id(
|
|
432
435
|
self, format: Literal["info", "dms"] = "info"
|
|
433
|
-
) -> dict[URIRef, ConceptualProperty] | dict[URIRef,
|
|
436
|
+
) -> dict[URIRef, ConceptualProperty] | dict[URIRef, PhysicalProperty]:
|
|
434
437
|
if format == "info":
|
|
435
|
-
return {prop.neatId: prop for prop in self.
|
|
438
|
+
return {prop.neatId: prop for prop in self.conceptual.properties if prop.neatId}
|
|
436
439
|
elif format == "dms":
|
|
437
|
-
return {prop.neatId: prop for prop in self.
|
|
440
|
+
return {prop.neatId: prop for prop in self.physical.properties if prop.neatId}
|
|
438
441
|
else:
|
|
439
442
|
raise NeatValueError(f"Invalid format: {format}")
|
|
440
443
|
|
|
441
444
|
@property
|
|
442
|
-
def
|
|
443
|
-
return {
|
|
445
|
+
def concepts_by_neat_id(self) -> dict[URIRef, Concept]:
|
|
446
|
+
return {concept.neatId: concept for concept in self.conceptual.concepts if concept.neatId}
|
|
444
447
|
|
|
445
448
|
@property
|
|
446
449
|
def multi_value_properties(self) -> list[ConceptualProperty]:
|
|
447
|
-
return [prop_ for prop_ in self.
|
|
450
|
+
return [prop_ for prop_ in self.conceptual.properties if isinstance(prop_.value_type, MultiValueTypeInfo)]
|
|
448
451
|
|
|
449
452
|
@property
|
|
450
453
|
def view_query_by_id(
|
|
451
454
|
self,
|
|
452
455
|
) -> "ViewQueryDict":
|
|
453
456
|
# Trigger error if any of these are missing
|
|
454
|
-
_ = self.
|
|
455
|
-
_ = self.
|
|
457
|
+
_ = self.conceptual
|
|
458
|
+
_ = self.physical
|
|
456
459
|
|
|
457
460
|
# caching results for faster access
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
461
|
+
concepts_by_neat_id = self._concept_by_neat_id
|
|
462
|
+
properties_by_concept = self.properties_by_concepts(include_ancestors=True)
|
|
463
|
+
conceptual_uri_by_view = self.conceptual_uri_by_view
|
|
464
|
+
conceptual_uri_by_property_by_view = self.conceptual_uri_by_property_by_view(include_ancestors=True)
|
|
465
|
+
conceptual_properties_by_neat_id = self._properties_by_neat_id()
|
|
463
466
|
|
|
464
467
|
query_configs = ViewQueryDict()
|
|
465
|
-
for view in self.
|
|
468
|
+
for view in self.physical.views:
|
|
466
469
|
# this entire block of sequential if statements checks:
|
|
467
|
-
# 1. connection of
|
|
468
|
-
# 2. correct paring of
|
|
469
|
-
# 3. connection of
|
|
470
|
+
# 1. connection of physical and conceptual data model
|
|
471
|
+
# 2. correct paring of conceptual and physical data model
|
|
472
|
+
# 3. connection of conceptual data model to instances
|
|
470
473
|
if (
|
|
471
|
-
(neat_id :=
|
|
472
|
-
and (
|
|
473
|
-
and (uri :=
|
|
474
|
+
(neat_id := conceptual_uri_by_view.get(view.view))
|
|
475
|
+
and (concept := concepts_by_neat_id.get(neat_id))
|
|
476
|
+
and (uri := concept.instance_source)
|
|
474
477
|
):
|
|
475
478
|
view_query = ViewQuery(
|
|
476
479
|
view_id=view.view.as_id(),
|
|
@@ -479,14 +482,14 @@ class RulesAnalysis:
|
|
|
479
482
|
# this is to encounter for special cases of e.g. space, startNode and endNode
|
|
480
483
|
property_renaming_config=(
|
|
481
484
|
{uri: prop_.property_ for prop_ in info_properties for uri in prop_.instance_source or []}
|
|
482
|
-
if (info_properties :=
|
|
485
|
+
if (info_properties := properties_by_concept.get(concept.concept))
|
|
483
486
|
else {}
|
|
484
487
|
),
|
|
485
488
|
)
|
|
486
489
|
|
|
487
|
-
if
|
|
488
|
-
for target_name, neat_id in
|
|
489
|
-
if (property_ :=
|
|
490
|
+
if conceptual_uri_by_property := conceptual_uri_by_property_by_view.get(view.view):
|
|
491
|
+
for target_name, neat_id in conceptual_uri_by_property.items():
|
|
492
|
+
if (property_ := conceptual_properties_by_neat_id.get(neat_id)) and (
|
|
490
493
|
uris := property_.instance_source
|
|
491
494
|
):
|
|
492
495
|
for uri in uris:
|
|
@@ -496,14 +499,14 @@ class RulesAnalysis:
|
|
|
496
499
|
|
|
497
500
|
return query_configs
|
|
498
501
|
|
|
499
|
-
def
|
|
500
|
-
"""Generate a MultiDiGraph from the
|
|
502
|
+
def _physical_di_graph(self, format: Literal["data-model", "implements"] = "data-model") -> nx.MultiDiGraph:
|
|
503
|
+
"""Generate a MultiDiGraph from the Physical Data Model."""
|
|
501
504
|
di_graph = nx.MultiDiGraph()
|
|
502
505
|
|
|
503
|
-
|
|
506
|
+
data_model = self.physical
|
|
504
507
|
|
|
505
508
|
# Add nodes and edges from Views sheet
|
|
506
|
-
for view in
|
|
509
|
+
for view in data_model.views:
|
|
507
510
|
di_graph.add_node(view.view.suffix, label=view.view.suffix)
|
|
508
511
|
|
|
509
512
|
if format == "implements" and view.implements:
|
|
@@ -518,7 +521,7 @@ class RulesAnalysis:
|
|
|
518
521
|
|
|
519
522
|
if format == "data-model":
|
|
520
523
|
# Add nodes and edges from Properties sheet
|
|
521
|
-
for prop_ in
|
|
524
|
+
for prop_ in data_model.properties:
|
|
522
525
|
if prop_.connection and isinstance(prop_.value_type, ViewEntity):
|
|
523
526
|
di_graph.add_node(prop_.view.suffix, label=prop_.view.suffix)
|
|
524
527
|
di_graph.add_node(prop_.value_type.suffix, label=prop_.value_type.suffix)
|
|
@@ -530,26 +533,26 @@ class RulesAnalysis:
|
|
|
530
533
|
|
|
531
534
|
return di_graph
|
|
532
535
|
|
|
533
|
-
def
|
|
534
|
-
"""Generate MultiDiGraph representing
|
|
536
|
+
def _conceptual_di_graph(self, format: Literal["data-model", "implements"] = "data-model") -> nx.MultiDiGraph:
|
|
537
|
+
"""Generate MultiDiGraph representing conceptual data model."""
|
|
535
538
|
|
|
536
|
-
|
|
539
|
+
data_model = self.conceptual
|
|
537
540
|
di_graph = nx.MultiDiGraph()
|
|
538
541
|
|
|
539
542
|
# Add nodes and edges from Views sheet
|
|
540
|
-
for
|
|
543
|
+
for concept in data_model.concepts:
|
|
541
544
|
# if possible use human readable label coming from the view name
|
|
542
545
|
|
|
543
546
|
di_graph.add_node(
|
|
544
|
-
|
|
545
|
-
label=
|
|
547
|
+
concept.concept.suffix,
|
|
548
|
+
label=concept.name or concept.concept.suffix,
|
|
546
549
|
)
|
|
547
550
|
|
|
548
|
-
if format == "implements" and
|
|
549
|
-
for parent in
|
|
551
|
+
if format == "implements" and concept.implements:
|
|
552
|
+
for parent in concept.implements:
|
|
550
553
|
di_graph.add_node(parent.suffix, label=parent.suffix)
|
|
551
554
|
di_graph.add_edge(
|
|
552
|
-
|
|
555
|
+
concept.concept.suffix,
|
|
553
556
|
parent.suffix,
|
|
554
557
|
label="implements",
|
|
555
558
|
dashes=True,
|
|
@@ -557,13 +560,13 @@ class RulesAnalysis:
|
|
|
557
560
|
|
|
558
561
|
if format == "data-model":
|
|
559
562
|
# Add nodes and edges from Properties sheet
|
|
560
|
-
for prop_ in
|
|
561
|
-
if isinstance(prop_.value_type,
|
|
562
|
-
di_graph.add_node(prop_.
|
|
563
|
+
for prop_ in data_model.properties:
|
|
564
|
+
if isinstance(prop_.value_type, ConceptEntity) and not isinstance(prop_.value_type, UnknownEntity):
|
|
565
|
+
di_graph.add_node(prop_.concept.suffix, label=prop_.concept.suffix)
|
|
563
566
|
di_graph.add_node(prop_.value_type.suffix, label=prop_.value_type.suffix)
|
|
564
567
|
|
|
565
568
|
di_graph.add_edge(
|
|
566
|
-
prop_.
|
|
569
|
+
prop_.concept.suffix,
|
|
567
570
|
prop_.value_type.suffix,
|
|
568
571
|
label=prop_.name or prop_.property_,
|
|
569
572
|
)
|