cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -8,11 +8,14 @@ from rdflib import Namespace, URIRef
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._constants import COGNITE_SPACES, DEFAULT_NAMESPACE
|
|
10
10
|
from cognite.neat.core._data_model.importers import DMSImporter
|
|
11
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
11
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
12
12
|
from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
|
|
13
13
|
from cognite.neat.core._data_model.models.data_types import Json
|
|
14
14
|
from cognite.neat.core._data_model.models.entities import UnknownEntity
|
|
15
|
-
from cognite.neat.core._data_model.transformers import
|
|
15
|
+
from cognite.neat.core._data_model.transformers import (
|
|
16
|
+
PhysicalToConceptual,
|
|
17
|
+
VerifyPhysicalDataModel,
|
|
18
|
+
)
|
|
16
19
|
from cognite.neat.core._issues import IssueList, NeatIssue, catch_warnings
|
|
17
20
|
from cognite.neat.core._issues.warnings import (
|
|
18
21
|
CDFAuthWarning,
|
|
@@ -47,8 +50,8 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
47
50
|
self._str_to_ideal_type = str_to_ideal_type
|
|
48
51
|
|
|
49
52
|
self._views: list[dm.View] | None = None
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
53
|
+
self._conceptual_data_model: ConceptualDataModel | None = None
|
|
54
|
+
self._physical_data_model: PhysicalDataModel | None = None
|
|
52
55
|
|
|
53
56
|
@classmethod
|
|
54
57
|
def from_data_model_id(
|
|
@@ -164,32 +167,32 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
164
167
|
self._issues.append(ResourceNotFoundWarning(dm_view, "view", data_model_id, "data model"))
|
|
165
168
|
return views
|
|
166
169
|
|
|
167
|
-
def
|
|
168
|
-
"""Returns the
|
|
169
|
-
if self.
|
|
170
|
-
self.
|
|
171
|
-
return self.
|
|
170
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
171
|
+
"""Returns the conceptual data model that the extractor uses."""
|
|
172
|
+
if self._conceptual_data_model is None:
|
|
173
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
174
|
+
return self._conceptual_data_model
|
|
172
175
|
|
|
173
|
-
def
|
|
174
|
-
"""Returns the
|
|
175
|
-
if self.
|
|
176
|
-
self.
|
|
177
|
-
return self.
|
|
176
|
+
def get_physical_data_model(self) -> PhysicalDataModel:
|
|
177
|
+
"""Returns the physical data model that the extractor uses."""
|
|
178
|
+
if self._physical_data_model is None:
|
|
179
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
180
|
+
return self._physical_data_model
|
|
178
181
|
|
|
179
182
|
def get_issues(self) -> IssueList:
|
|
180
183
|
"""Returns the issues that occurred during the extraction."""
|
|
181
184
|
return self._issues
|
|
182
185
|
|
|
183
|
-
def
|
|
184
|
-
# The
|
|
186
|
+
def _create_data_models(self) -> tuple[ConceptualDataModel, PhysicalDataModel]:
|
|
187
|
+
# The physical and conceptual data model must be created together to link them property.
|
|
185
188
|
importer = DMSImporter.from_data_model(self._client, self._data_model)
|
|
186
|
-
|
|
187
|
-
if self._unpack_json and (
|
|
188
|
-
# Drop the JSON properties from the
|
|
189
|
+
imported_physical_data_model = importer.to_data_model()
|
|
190
|
+
if self._unpack_json and (unverified_physical_data_model := imported_physical_data_model.unverified_data_model):
|
|
191
|
+
# Drop the JSON properties from the physical data model as these are no longer valid.
|
|
189
192
|
json_name = Json().name # To avoid instantiating Json multiple times.
|
|
190
|
-
|
|
193
|
+
unverified_physical_data_model.properties = [
|
|
191
194
|
prop
|
|
192
|
-
for prop in
|
|
195
|
+
for prop in unverified_physical_data_model.properties
|
|
193
196
|
if not (
|
|
194
197
|
(
|
|
195
198
|
isinstance(prop.value_type, Json)
|
|
@@ -202,23 +205,28 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
202
205
|
|
|
203
206
|
with catch_warnings() as issues:
|
|
204
207
|
# Any errors occur will be raised and caught outside the extractor.
|
|
205
|
-
|
|
206
|
-
|
|
208
|
+
verified_physical_data_model = VerifyPhysicalDataModel(client=self._client).transform(
|
|
209
|
+
imported_physical_data_model
|
|
210
|
+
)
|
|
211
|
+
verified_conceptual_data_model = PhysicalToConceptual(self._namespace).transform(
|
|
212
|
+
verified_physical_data_model
|
|
213
|
+
)
|
|
207
214
|
|
|
208
|
-
# We need to sync the metadata between the two
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
215
|
+
# We need to sync the metadata between the two data model, such that the
|
|
216
|
+
# `.sync_with_conceptual_data_model` method works.
|
|
217
|
+
verified_conceptual_data_model.metadata.physical = verified_physical_data_model.metadata.identifier
|
|
218
|
+
verified_physical_data_model.metadata.conceptual = verified_conceptual_data_model.metadata.identifier
|
|
219
|
+
verified_physical_data_model.sync_with_conceptual_data_model(verified_conceptual_data_model)
|
|
212
220
|
|
|
213
|
-
# Adding startNode and endNode to the
|
|
214
|
-
|
|
221
|
+
# Adding startNode and endNode to the conceptual data model for views that are used for edges.
|
|
222
|
+
concepts_by_prefix = {concept.concept.prefix: concept for concept in verified_conceptual_data_model.concepts}
|
|
215
223
|
for view in self._model_views:
|
|
216
|
-
if view.used_for == "edge" and view.external_id in
|
|
217
|
-
cls_ =
|
|
224
|
+
if view.used_for == "edge" and view.external_id in concepts_by_prefix:
|
|
225
|
+
cls_ = concepts_by_prefix[view.external_id]
|
|
218
226
|
for property_ in ("startNode", "endNode"):
|
|
219
|
-
|
|
227
|
+
verified_conceptual_data_model.properties.append(
|
|
220
228
|
ConceptualProperty(
|
|
221
|
-
|
|
229
|
+
concept=cls_.concept,
|
|
222
230
|
property_=property_,
|
|
223
231
|
value_type=UnknownEntity(),
|
|
224
232
|
min_count=0,
|
|
@@ -227,4 +235,4 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
227
235
|
)
|
|
228
236
|
|
|
229
237
|
self._issues.extend(issues)
|
|
230
|
-
return
|
|
238
|
+
return verified_conceptual_data_model, verified_physical_data_model
|
|
@@ -12,12 +12,12 @@ import pandas as pd
|
|
|
12
12
|
from rdflib import RDF, Literal, Namespace, URIRef
|
|
13
13
|
|
|
14
14
|
from cognite.neat.core._data_model._constants import EntityTypes
|
|
15
|
-
from cognite.neat.core._data_model.analysis import
|
|
16
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
15
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
16
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
17
17
|
from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
|
|
18
18
|
from cognite.neat.core._data_model.models.data_types import DataType
|
|
19
|
-
from cognite.neat.core._data_model.models.entities import
|
|
20
|
-
from cognite.neat.core._data_model.transformers import
|
|
19
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity
|
|
20
|
+
from cognite.neat.core._data_model.transformers import SubsetConceptualDataModel
|
|
21
21
|
from cognite.neat.core._shared import Triple
|
|
22
22
|
from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
|
|
23
23
|
|
|
@@ -29,8 +29,8 @@ class MockGraphGenerator(BaseExtractor):
|
|
|
29
29
|
Class used to generate mock graph data for purposes of testing of NEAT.
|
|
30
30
|
|
|
31
31
|
Args:
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
data_model: Data model defining the concepts with their properties.
|
|
33
|
+
concept_count: Target concept count for each concept/class in the data model
|
|
34
34
|
stop_on_exception: To stop if exception is encountered or not, default is False
|
|
35
35
|
allow_isolated_classes: To allow generation of instances for classes that are not
|
|
36
36
|
connected to any other class, default is True
|
|
@@ -38,127 +38,131 @@ class MockGraphGenerator(BaseExtractor):
|
|
|
38
38
|
|
|
39
39
|
def __init__(
|
|
40
40
|
self,
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
data_model: ConceptualDataModel | PhysicalDataModel,
|
|
42
|
+
concept_count: dict[str | ConceptEntity, int] | None = None,
|
|
43
43
|
stop_on_exception: bool = False,
|
|
44
44
|
allow_isolated_classes: bool = True,
|
|
45
45
|
):
|
|
46
|
-
if isinstance(
|
|
46
|
+
if isinstance(data_model, PhysicalDataModel):
|
|
47
47
|
# fixes potential issues with circular dependencies
|
|
48
|
-
from cognite.neat.core._data_model.transformers import
|
|
48
|
+
from cognite.neat.core._data_model.transformers import PhysicalToConceptual
|
|
49
49
|
|
|
50
|
-
self.
|
|
51
|
-
elif isinstance(
|
|
52
|
-
self.
|
|
50
|
+
self.data_model = PhysicalToConceptual().transform(data_model)
|
|
51
|
+
elif isinstance(data_model, ConceptualDataModel):
|
|
52
|
+
self.data_model = data_model
|
|
53
53
|
else:
|
|
54
|
-
raise ValueError("
|
|
54
|
+
raise ValueError("Data model must be of type Conceptual or Physical!")
|
|
55
55
|
|
|
56
|
-
if not
|
|
57
|
-
self.
|
|
58
|
-
|
|
56
|
+
if not concept_count:
|
|
57
|
+
self.concept_count = {
|
|
58
|
+
concept: 1 for concept in DataModelAnalysis(self.data_model).defined_concepts(include_ancestors=True)
|
|
59
59
|
}
|
|
60
|
-
elif all(isinstance(key, str) for key in
|
|
61
|
-
self.
|
|
62
|
-
|
|
60
|
+
elif all(isinstance(key, str) for key in concept_count.keys()):
|
|
61
|
+
self.concept_count = {
|
|
62
|
+
ConceptEntity.load(f"{self.data_model.metadata.prefix}:{key}"): value
|
|
63
|
+
for key, value in concept_count.items()
|
|
63
64
|
}
|
|
64
|
-
elif all(isinstance(key,
|
|
65
|
-
self.
|
|
65
|
+
elif all(isinstance(key, ConceptEntity) for key in concept_count.keys()):
|
|
66
|
+
self.concept_count = cast(dict[ConceptEntity, int], concept_count)
|
|
66
67
|
else:
|
|
67
|
-
raise ValueError("Class count keys must be of type str! or
|
|
68
|
+
raise ValueError("Class count keys must be of type str! or ConceptEntity! or empty dict!")
|
|
68
69
|
|
|
69
70
|
self.stop_on_exception = stop_on_exception
|
|
70
71
|
self.allow_isolated_classes = allow_isolated_classes
|
|
71
72
|
|
|
72
73
|
def extract(self) -> list[Triple]:
|
|
73
|
-
"""Generate mock triples based on data model
|
|
74
|
-
of
|
|
74
|
+
"""Generate mock triples based on data model and desired number
|
|
75
|
+
of concept instances
|
|
75
76
|
|
|
76
77
|
Returns:
|
|
77
78
|
List of RDF triples, represented as tuples `(subject, predicate, object)`, that define data model instances
|
|
78
79
|
"""
|
|
79
80
|
return generate_triples(
|
|
80
|
-
self.
|
|
81
|
-
self.
|
|
81
|
+
self.data_model,
|
|
82
|
+
self.concept_count,
|
|
82
83
|
stop_on_exception=self.stop_on_exception,
|
|
83
|
-
|
|
84
|
+
allow_isolated_concepts=self.allow_isolated_classes,
|
|
84
85
|
)
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
def generate_triples(
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
data_model: ConceptualDataModel,
|
|
90
|
+
concept_count: dict[ConceptEntity, int],
|
|
90
91
|
stop_on_exception: bool = False,
|
|
91
|
-
|
|
92
|
+
allow_isolated_concepts: bool = True,
|
|
92
93
|
) -> list[Triple]:
|
|
93
|
-
"""Generate mock triples based on data model defined
|
|
94
|
+
"""Generate mock triples based on the conceptual data model defined and desired number
|
|
94
95
|
of class instances
|
|
95
96
|
|
|
96
97
|
Args:
|
|
97
|
-
|
|
98
|
-
|
|
98
|
+
data_model : Data model
|
|
99
|
+
concept_count: Target concept count for each class in the data model
|
|
99
100
|
stop_on_exception: To stop if exception is encountered or not, default is False
|
|
100
|
-
|
|
101
|
+
allow_isolated_concepts: To allow generation of instances for classes that are not
|
|
101
102
|
connected to any other class, default is True
|
|
102
103
|
|
|
103
104
|
Returns:
|
|
104
105
|
List of RDF triples, represented as tuples `(subject, predicate, object)`, that define data model instances
|
|
105
106
|
"""
|
|
106
107
|
|
|
107
|
-
namespace =
|
|
108
|
-
analysis =
|
|
109
|
-
|
|
108
|
+
namespace = data_model.metadata.namespace
|
|
109
|
+
analysis = DataModelAnalysis(data_model)
|
|
110
|
+
defined_concepts = analysis.defined_concepts(include_ancestors=True)
|
|
110
111
|
|
|
111
|
-
if
|
|
112
|
-
msg =
|
|
112
|
+
if non_existing_concepts := set(concept_count.keys()) - defined_concepts:
|
|
113
|
+
msg = (
|
|
114
|
+
f"Concept count contains concepts {non_existing_concepts} for which"
|
|
115
|
+
" properties are not defined in Data Model!"
|
|
116
|
+
)
|
|
113
117
|
if stop_on_exception:
|
|
114
118
|
raise ValueError(msg)
|
|
115
119
|
else:
|
|
116
120
|
msg += " These classes will be ignored."
|
|
117
121
|
warnings.warn(msg, stacklevel=2)
|
|
118
|
-
for
|
|
119
|
-
|
|
122
|
+
for concept in non_existing_concepts:
|
|
123
|
+
concept_count.pop(concept)
|
|
120
124
|
|
|
121
125
|
# Subset data model to only classes that are defined in class count
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if
|
|
125
|
-
else
|
|
126
|
+
data_model = (
|
|
127
|
+
SubsetConceptualDataModel(concepts=set(concept_count.keys())).transform(data_model)
|
|
128
|
+
if defined_concepts != set(concept_count.keys())
|
|
129
|
+
else data_model
|
|
126
130
|
)
|
|
127
131
|
|
|
128
|
-
|
|
132
|
+
concept_linkage = analysis.concept_linkage().to_pandas()
|
|
129
133
|
|
|
130
134
|
# Remove one of symmetric pairs from class linkage to maintain proper linking
|
|
131
135
|
# among instances of symmetrically linked classes
|
|
132
|
-
if sym_pairs := analysis.
|
|
133
|
-
|
|
136
|
+
if sym_pairs := analysis.symmetrically_connected_concepts():
|
|
137
|
+
concept_linkage = _remove_higher_occurring_sym_pair(concept_linkage, sym_pairs)
|
|
134
138
|
|
|
135
139
|
# Remove any of symmetric pairs containing classes that are not present class count
|
|
136
|
-
|
|
140
|
+
concept_linkage = _remove_non_requested_sym_pairs(concept_linkage, concept_count)
|
|
137
141
|
|
|
138
142
|
# Generate generation order for classes instances
|
|
139
|
-
generation_order = _prettify_generation_order(_get_generation_order(
|
|
143
|
+
generation_order = _prettify_generation_order(_get_generation_order(concept_linkage))
|
|
140
144
|
|
|
141
145
|
# Generated simple view of data model
|
|
142
|
-
|
|
146
|
+
properties_by_concepts = analysis.properties_by_concepts(include_ancestors=True)
|
|
143
147
|
|
|
144
148
|
# pregenerate instance ids for each remaining class
|
|
145
149
|
instance_ids = {
|
|
146
|
-
key: [URIRef(namespace[f"{key.suffix}-{i + 1}"]) for i in range(value)] for key, value in
|
|
150
|
+
key: [URIRef(namespace[f"{key.suffix}-{i + 1}"]) for i in range(value)] for key, value in concept_count.items()
|
|
147
151
|
}
|
|
148
152
|
|
|
149
153
|
# create triple for each class instance defining its type
|
|
150
154
|
triples: list[Triple] = []
|
|
151
|
-
for
|
|
155
|
+
for concept in concept_count:
|
|
152
156
|
triples += [
|
|
153
|
-
(
|
|
154
|
-
for
|
|
157
|
+
(concept_instance_id, RDF.type, URIRef(namespace[str(concept.suffix)]))
|
|
158
|
+
for concept_instance_id in instance_ids[concept]
|
|
155
159
|
]
|
|
156
160
|
|
|
157
161
|
# generate triples for connected classes
|
|
158
|
-
for
|
|
162
|
+
for concept in generation_order:
|
|
159
163
|
triples += _generate_triples_per_class(
|
|
160
|
-
|
|
161
|
-
|
|
164
|
+
concept,
|
|
165
|
+
properties_by_concepts,
|
|
162
166
|
sym_pairs,
|
|
163
167
|
instance_ids,
|
|
164
168
|
namespace,
|
|
@@ -166,11 +170,11 @@ def generate_triples(
|
|
|
166
170
|
)
|
|
167
171
|
|
|
168
172
|
# generate triples for isolated classes
|
|
169
|
-
if
|
|
170
|
-
for
|
|
173
|
+
if allow_isolated_concepts:
|
|
174
|
+
for concept in set(concept_count.keys()) - set(generation_order):
|
|
171
175
|
triples += _generate_triples_per_class(
|
|
172
|
-
|
|
173
|
-
|
|
176
|
+
concept,
|
|
177
|
+
properties_by_concepts,
|
|
174
178
|
sym_pairs,
|
|
175
179
|
instance_ids,
|
|
176
180
|
namespace,
|
|
@@ -181,11 +185,11 @@ def generate_triples(
|
|
|
181
185
|
|
|
182
186
|
|
|
183
187
|
def _get_generation_order(
|
|
184
|
-
|
|
188
|
+
concept_linkage: pd.DataFrame,
|
|
185
189
|
parent_col: str = "source_class",
|
|
186
190
|
child_col: str = "target_class",
|
|
187
191
|
) -> dict:
|
|
188
|
-
parent_child_list: list[list[str]] =
|
|
192
|
+
parent_child_list: list[list[str]] = concept_linkage[[parent_col, child_col]].values.tolist() # type: ignore[assignment]
|
|
189
193
|
# Build a directed graph and a list of all names that have no parent
|
|
190
194
|
graph: dict[str, set] = {name: set() for tup in parent_child_list for name in tup}
|
|
191
195
|
has_parent: dict[str, bool] = {name: False for tup in parent_child_list for name in tup}
|
|
@@ -217,44 +221,44 @@ def _prettify_generation_order(generation_order: dict, depth: dict | None = None
|
|
|
217
221
|
|
|
218
222
|
|
|
219
223
|
def _remove_higher_occurring_sym_pair(
|
|
220
|
-
|
|
224
|
+
concept_linkage: pd.DataFrame, sym_pairs: set[tuple[ConceptEntity, ConceptEntity]]
|
|
221
225
|
) -> pd.DataFrame:
|
|
222
226
|
"""Remove symmetric pair which is higher in occurrence."""
|
|
223
227
|
rows_to_remove = set()
|
|
224
228
|
for source, target in sym_pairs:
|
|
225
|
-
first_sym_property_occurrence =
|
|
226
|
-
(
|
|
229
|
+
first_sym_property_occurrence = concept_linkage[
|
|
230
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
227
231
|
].max_occurrence.values[0]
|
|
228
|
-
second_sym_property_occurrence =
|
|
229
|
-
(
|
|
232
|
+
second_sym_property_occurrence = concept_linkage[
|
|
233
|
+
(concept_linkage.source_class == target) & (concept_linkage.target_class == source)
|
|
230
234
|
].max_occurrence.values[0]
|
|
231
235
|
|
|
232
236
|
if first_sym_property_occurrence is None:
|
|
233
237
|
# this means that source occurrence is unbounded
|
|
234
|
-
index =
|
|
235
|
-
(
|
|
238
|
+
index = concept_linkage[
|
|
239
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
236
240
|
].index.values[0]
|
|
237
241
|
elif second_sym_property_occurrence is None or (
|
|
238
242
|
first_sym_property_occurrence <= second_sym_property_occurrence
|
|
239
243
|
and second_sym_property_occurrence > first_sym_property_occurrence
|
|
240
244
|
):
|
|
241
245
|
# this means that target occurrence is unbounded
|
|
242
|
-
index =
|
|
243
|
-
(
|
|
246
|
+
index = concept_linkage[
|
|
247
|
+
(concept_linkage.source_class == target) & (concept_linkage.target_class == source)
|
|
244
248
|
].index.values[0]
|
|
245
249
|
else:
|
|
246
|
-
index =
|
|
247
|
-
(
|
|
250
|
+
index = concept_linkage[
|
|
251
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
248
252
|
].index.values[0]
|
|
249
253
|
rows_to_remove.add(index)
|
|
250
254
|
|
|
251
|
-
return
|
|
255
|
+
return concept_linkage.drop(list(rows_to_remove))
|
|
252
256
|
|
|
253
257
|
|
|
254
|
-
def _remove_non_requested_sym_pairs(class_linkage: pd.DataFrame,
|
|
258
|
+
def _remove_non_requested_sym_pairs(class_linkage: pd.DataFrame, concept_count: dict) -> pd.DataFrame:
|
|
255
259
|
"""Remove symmetric pairs which classes are not found in class count."""
|
|
256
|
-
rows_to_remove = set(class_linkage[~(class_linkage["source_class"].isin(set(
|
|
257
|
-
rows_to_remove |= set(class_linkage[~(class_linkage["target_class"].isin(set(
|
|
260
|
+
rows_to_remove = set(class_linkage[~(class_linkage["source_class"].isin(set(concept_count.keys())))].index.values)
|
|
261
|
+
rows_to_remove |= set(class_linkage[~(class_linkage["target_class"].isin(set(concept_count.keys())))].index.values)
|
|
258
262
|
|
|
259
263
|
return class_linkage.drop(list(rows_to_remove))
|
|
260
264
|
|
|
@@ -293,41 +297,41 @@ def _generate_mock_data_property_triples(
|
|
|
293
297
|
|
|
294
298
|
|
|
295
299
|
def _generate_mock_object_property_triples(
|
|
296
|
-
|
|
300
|
+
concept: ConceptEntity,
|
|
297
301
|
property_definition: ConceptualProperty,
|
|
298
|
-
|
|
299
|
-
sym_pairs: set[tuple[
|
|
300
|
-
instance_ids: dict[
|
|
302
|
+
concept_property_pairs: dict[ConceptEntity, list[ConceptualProperty]],
|
|
303
|
+
sym_pairs: set[tuple[ConceptEntity, ConceptEntity]],
|
|
304
|
+
instance_ids: dict[ConceptEntity, list[URIRef]],
|
|
301
305
|
namespace: Namespace,
|
|
302
306
|
stop_on_exception: bool,
|
|
303
307
|
) -> list[tuple[URIRef, URIRef, URIRef]]:
|
|
304
308
|
"""Generates triples for object properties."""
|
|
305
309
|
if property_definition.value_type not in instance_ids:
|
|
306
|
-
msg = f"
|
|
310
|
+
msg = f"Concept {property_definition.value_type} not found in concept count! "
|
|
307
311
|
if stop_on_exception:
|
|
308
312
|
raise ValueError(msg)
|
|
309
313
|
else:
|
|
310
314
|
msg += (
|
|
311
315
|
f"Skipping creating triples for property {property_definition.name} "
|
|
312
|
-
f"of
|
|
316
|
+
f"of concept {concept.suffix} which expects values of this type!"
|
|
313
317
|
)
|
|
314
318
|
warnings.warn(msg, stacklevel=2)
|
|
315
319
|
return []
|
|
316
320
|
|
|
317
321
|
# Handling symmetric property
|
|
318
322
|
|
|
319
|
-
if tuple((
|
|
320
|
-
|
|
323
|
+
if tuple((concept, property_definition.value_type)) in sym_pairs:
|
|
324
|
+
symmetric_concept_properties = concept_property_pairs[cast(ConceptEntity, property_definition.value_type)]
|
|
321
325
|
candidates = list(
|
|
322
326
|
filter(
|
|
323
|
-
lambda instance: instance.value_type ==
|
|
324
|
-
|
|
327
|
+
lambda instance: instance.value_type == concept,
|
|
328
|
+
symmetric_concept_properties,
|
|
325
329
|
)
|
|
326
330
|
)
|
|
327
331
|
symmetric_property = candidates[0]
|
|
328
332
|
if len(candidates) > 1:
|
|
329
333
|
warnings.warn(
|
|
330
|
-
f"Multiple symmetric properties found for
|
|
334
|
+
f"Multiple symmetric properties found for concept {property_definition.value_type}! "
|
|
331
335
|
f"Only one will be used for creating symmetric triples.",
|
|
332
336
|
stacklevel=2,
|
|
333
337
|
)
|
|
@@ -336,9 +340,9 @@ def _generate_mock_object_property_triples(
|
|
|
336
340
|
|
|
337
341
|
triples = []
|
|
338
342
|
|
|
339
|
-
for i, source in enumerate(instance_ids[
|
|
340
|
-
target = instance_ids[cast(
|
|
341
|
-
i % len(instance_ids[cast(
|
|
343
|
+
for i, source in enumerate(instance_ids[concept]):
|
|
344
|
+
target = instance_ids[cast(ConceptEntity, property_definition.value_type)][
|
|
345
|
+
i % len(instance_ids[cast(ConceptEntity, property_definition.value_type)])
|
|
342
346
|
]
|
|
343
347
|
triples += [
|
|
344
348
|
(
|
|
@@ -358,26 +362,26 @@ def _generate_mock_object_property_triples(
|
|
|
358
362
|
]
|
|
359
363
|
|
|
360
364
|
if symmetric_property:
|
|
361
|
-
|
|
365
|
+
concept_property_pairs[cast(ConceptEntity, property_definition.value_type)].remove(symmetric_property)
|
|
362
366
|
|
|
363
367
|
return triples
|
|
364
368
|
|
|
365
369
|
|
|
366
370
|
def _generate_triples_per_class(
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
sym_pairs: set[tuple[
|
|
370
|
-
instance_ids: dict[
|
|
371
|
+
concept: ConceptEntity,
|
|
372
|
+
concept_properties_pairs: dict[ConceptEntity, list[ConceptualProperty]],
|
|
373
|
+
sym_pairs: set[tuple[ConceptEntity, ConceptEntity]],
|
|
374
|
+
instance_ids: dict[ConceptEntity, list[URIRef]],
|
|
371
375
|
namespace: Namespace,
|
|
372
376
|
stop_on_exception: bool,
|
|
373
377
|
) -> list[Triple]:
|
|
374
378
|
"""Generate triples for a given class."""
|
|
375
379
|
triples: list[Triple] = []
|
|
376
380
|
|
|
377
|
-
for property_ in
|
|
381
|
+
for property_ in concept_properties_pairs[concept]:
|
|
378
382
|
if property_.type_ == EntityTypes.data_property:
|
|
379
383
|
triples += _generate_mock_data_property_triples(
|
|
380
|
-
instance_ids[
|
|
384
|
+
instance_ids[concept],
|
|
381
385
|
property_.property_,
|
|
382
386
|
namespace,
|
|
383
387
|
cast(DataType, property_.value_type),
|
|
@@ -385,9 +389,9 @@ def _generate_triples_per_class(
|
|
|
385
389
|
|
|
386
390
|
elif property_.type_ == EntityTypes.object_property:
|
|
387
391
|
triples += _generate_mock_object_property_triples(
|
|
388
|
-
|
|
392
|
+
concept,
|
|
389
393
|
property_,
|
|
390
|
-
|
|
394
|
+
concept_properties_pairs,
|
|
391
395
|
sym_pairs,
|
|
392
396
|
instance_ids,
|
|
393
397
|
namespace,
|
|
@@ -8,7 +8,7 @@ from cognite.client.data_classes.capabilities import Capability
|
|
|
8
8
|
|
|
9
9
|
from cognite.neat.core._issues import IssueList, NeatIssue
|
|
10
10
|
from cognite.neat.core._issues.errors import AuthorizationError
|
|
11
|
-
from cognite.neat.core._store import
|
|
11
|
+
from cognite.neat.core._store import NeatInstanceStore
|
|
12
12
|
from cognite.neat.core._utils.auxiliary import class_html_doc
|
|
13
13
|
from cognite.neat.core._utils.upload import UploadResult, UploadResultList
|
|
14
14
|
|
|
@@ -29,8 +29,8 @@ class BaseLoader(ABC, Generic[T_Output]):
|
|
|
29
29
|
_new_line = "\n"
|
|
30
30
|
_encoding = "utf-8"
|
|
31
31
|
|
|
32
|
-
def __init__(self,
|
|
33
|
-
self.
|
|
32
|
+
def __init__(self, instance_store: NeatInstanceStore):
|
|
33
|
+
self.instance_store = instance_store
|
|
34
34
|
|
|
35
35
|
@abstractmethod
|
|
36
36
|
def write_to_file(self, filepath: Path) -> None:
|